2019-12-07T19:33:00.924+0000 7f686bd1cbc0 0 ceph version 15.0.0-8038-g503c226 (503c226f594b239359b549e9328eeb7369e666df) octopus (dev), process ceph-osd, pid 33916 2019-12-07T19:33:00.927+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) set_cache_shards 1 2019-12-07T19:33:00.927+0000 7f686bd1cbc0 -1 auth: error reading file: /var/lib/ceph/osd/ceph-0/keyring: can't open /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory 2019-12-07T19:33:00.930+0000 7f686bd1cbc0 -1 created new key in keyring /var/lib/ceph/osd/ceph-0/keyring 2019-12-07T19:33:00.930+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) mkfs path /var/lib/ceph/osd/ceph-0 2019-12-07T19:33:00.930+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:00.930+0000 7f686bd1cbc0 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2019-12-07T19:33:00.930+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:00.930+0000 7f686bd1cbc0 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2019-12-07T19:33:00.930+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:00.930+0000 7f686bd1cbc0 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2019-12-07T19:33:00.937+0000 7f686bd1cbc0 -1 bluestore(/var/lib/ceph/osd/ceph-0) _read_fsid unparsable uuid 2019-12-07T19:33:00.937+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) mkfs generated fsid b39c74bd-b656-4393-a258-c2eebf51a555 2019-12-07T19:33:00.937+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _setup_block_symlink_or_file name block path size 96636764160 create=1 2019-12-07T19:33:00.937+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _setup_block_symlink_or_file resized block file to 90 GiB 2019-12-07T19:33:00.937+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _setup_block_symlink_or_file name block.wal path size 100663296 create=0 2019-12-07T19:33:00.937+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _setup_block_symlink_or_file name block.db path size 0 create=0 2019-12-07T19:33:00.937+0000 7f686bd1cbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:00.937+0000 7f686bd1cbc0 1 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:00.937+0000 7f686bd1cbc0 0 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:00.937+0000 7f686bd1cbc0 1 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:00.937+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _write_bdev_label path /var/lib/ceph/osd/ceph-0/block label bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 0 meta) 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db kv_backend = rocksdb 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db do_bluefs = 1 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db initializing bluefs 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 1 bdev(0x55c5e4298380 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 0 bdev(0x55c5e4298380 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 1 bdev(0x55c5e4298380 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block size 90 GiB 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 1 bluefs _add_block_extent bdev 1 0xacccc0000~e6670000 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 bluefs _add_block_extent done 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log want_seq 0 txn(seq 0 len 0x0 crc 0xffffffff) not dirty, dirty_files empty, no-op 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 1 bluefs mkfs osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 20 bluefs _init_alloc 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_add_free done 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 1 bluefs mkfs uuid db82411a-1632-4be0-8c82-a647203179ea 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 bluefs _allocate len 0x400000 from 0 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 bluefs _allocate len 0x400000 from 1 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 allocate 0x400000/10000,400000,0 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 allocate extent: 0xacccc0000~400000/10000,400000,0 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 20 bluefs mkfs op_alloc_add 1 0xacccc0000~e6670000 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 0x0~1000 to file(ino 1 size 0x0 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bf900 pos 0x0 0x0~1000 to file(ino 1 size 0x0 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x0 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bf900 pos now 0x1000 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:00.938+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bf900 2019-12-07T19:33:00.939+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bf900 done in 0.000259 2019-12-07T19:33:00.939+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.939+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 1 2019-12-07T19:33:00.939+0000 7f686bd1cbc0 10 bluefs _write_super super block length(encoded): 102 2019-12-07T19:33:00.939+0000 7f686bd1cbc0 10 bluefs _write_super superblock 1 2019-12-07T19:33:00.939+0000 7f686bd1cbc0 10 bluefs _write_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.939+0000 7f686bd1cbc0 20 bluefs _write_super v 1 crc 0x94a25aa6 offset 0x1000 2019-12-07T19:33:00.939+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.941+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e37bf900 type 0 2019-12-07T19:33:00.941+0000 7f686bd1cbc0 20 bluefs _stop_alloc 2019-12-07T19:33:00.941+0000 7f686bd1cbc0 1 fbmap_alloc 0x55c5e364c900 shutdown 2019-12-07T19:33:00.941+0000 7f686bd1cbc0 10 bluefs mkfs success 2019-12-07T19:33:00.941+0000 7f686bd1cbc0 1 bluefs mount 2019-12-07T19:33:00.941+0000 7f686bd1cbc0 10 bluefs _open_super 2019-12-07T19:33:00.947+0000 7f686bd1cbc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:00.947+0000 7f686bd1cbc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.947+0000 7f686bd1cbc0 20 bluefs _init_alloc 2019-12-07T19:33:00.947+0000 7f686bd1cbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:00.947+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:00.947+0000 7f686bd1cbc0 10 bluefs _replay 2019-12-07T19:33:00.947+0000 7f686bd1cbc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.947+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.947+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_add_free done 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluefs _replay 0x1000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid db82411a-1632-4be0-8c82-a647203179ea 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluefs _replay log file size was 0x1000 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluefs _replay done 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_rm_free done 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluefs mount log write pos set to 0x1000 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluefs mkdir db 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluefs mkdir db.slow 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluefs mkdir db.wal 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family L: 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family M: 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family P: 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options using custom Env 0x55c5e4285b60 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:00.949+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Write Ahead Log file in db.wal: 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.create_if_missing: 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.env: 0x55c5e4285b60 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.info_log: 0x55c5e4284f40 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.write_buffer_manager: 0x55c5e4263d10 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs mkdir db.wal 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs mkdir db 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs mkdir db 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs mkdir db.wal 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs dir_exists db.wal = 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs lock_file db/LOCK 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs lock_file dir db (0x55c5e4288a20) file LOCK not found, creating 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs lock_file locked file(ino 2 size 0x0 mtime 2019-12-07T19:33:00.950897+0000 allocated 0 extents []) with 0x55c5e36182c0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs stat db/CURRENT 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs stat dir db (0x55c5e4288a20) file CURRENT not found 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl_open.cc:228] Creating manifest 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs open_for_write db/MANIFEST-000001 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs open_for_write mapping db/MANIFEST-000001 vsel_hint 0x1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs open_for_write h 0x55c5e37bf900 on file(ino 3 size 0x0 mtime 2019-12-07T19:33:00.950915+0000 allocated 0 extents []) 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 ignoring, length 13 < min_flush_size 524288 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 ignoring, length 13 < min_flush_size 524288 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs _fsync 0x55c5e37bf900 file(ino 3 size 0x0 mtime 2019-12-07T19:33:00.950915+0000 allocated 0 extents []) 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 0x0~d to file(ino 3 size 0x0 mtime 2019-12-07T19:33:00.950915+0000 allocated 0 extents []) 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bf900 pos 0x0 0x0~d to file(ino 3 size 0x0 mtime 2019-12-07T19:33:00.950915+0000 allocated 0 extents []) 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs _allocate len 0xd from 1 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 allocate extent: 0xacd0c0000~10000/10000,10000,0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs _flush_range dirty_seq = 2 (was clean) 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacd0c0000~10000 x_off 0x0 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs _flush_range caching tail of 0xd and padding block with 0xff3 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bf900 pos now 0xd 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bf900 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bf900 done in 0.000000 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs _fsync file metadata was dirty (2) on file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]), flushing log 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 2 len 0xa5 crc 0x5d8d242d) 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfb80 0x1000~1000 to file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfb80 pos 0x1000 0x1000~1000 to file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1000 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfb80 pos now 0x2000 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 done in 0.000024 2019-12-07T19:33:00.950+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 2 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 10 bluefs _truncate 0xd file file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e37bf900 type 0 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 10 bluefs open_for_write db/000001.dbtmp 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 20 bluefs open_for_write mapping db/000001.dbtmp vsel_hint 0x1 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 10 bluefs open_for_write h 0x55c5e37bf900 on file(ino 4 size 0x0 mtime 2019-12-07T19:33:00.952418+0000 allocated 0 extents []) 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 10 bluefs _fsync 0x55c5e37bf900 file(ino 4 size 0x0 mtime 2019-12-07T19:33:00.952418+0000 allocated 0 extents []) 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 0x0~10 to file(ino 4 size 0x0 mtime 2019-12-07T19:33:00.952418+0000 allocated 0 extents []) 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bf900 pos 0x0 0x0~10 to file(ino 4 size 0x0 mtime 2019-12-07T19:33:00.952418+0000 allocated 0 extents []) 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 10 bluefs _allocate len 0x10 from 1 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 allocate extent: 0xacd0d0000~10000/10000,10000,0 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 20 bluefs _flush_range dirty_seq = 3 (was clean) 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacd0d0000~10000 x_off 0x0 2019-12-07T19:33:00.951+0000 7f686bd1cbc0 20 bluefs _flush_range caching tail of 0x10 and padding block with 0xff0 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bf900 pos now 0x10 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bf900 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bf900 done in 0.000000 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs _fsync file metadata was dirty (3) on file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]), flushing log 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 3 len 0x57 crc 0x27b7df6c) 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfb80 0x2000~1000 to file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfb80 pos 0x2000 0x2000~1000 to file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x2000 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfb80 pos now 0x3000 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 done in 0.000032 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 3 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e37bf900 type 0 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs rename db/000001.dbtmp -> db/CURRENT 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs rename db/CURRENT file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs sync_metadata 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfb80 0x3000~1000 to file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfb80 pos 0x3000 0x3000~1000 to file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x3000 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfb80 pos now 0x4000 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 done in 0.000254 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 4 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs sync_metadata done in 0.000401 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs _should_compact_log current 0x4000 expected 3000 ratio 1.33333 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 10 bluefs stat db/IDENTITY 2019-12-07T19:33:00.952+0000 7f686bd1cbc0 20 bluefs stat dir db (0x55c5e4288a20) file IDENTITY not found 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs open_for_write db/000000.dbtmp 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 20 bluefs open_for_write mapping db/000000.dbtmp vsel_hint 0x1 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs open_for_write h 0x55c5e37bf900 on file(ino 5 size 0x0 mtime 2019-12-07T19:33:00.953767+0000 allocated 0 extents []) 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs _fsync 0x55c5e37bf900 file(ino 5 size 0x0 mtime 2019-12-07T19:33:00.953767+0000 allocated 0 extents []) 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 0x0~25 to file(ino 5 size 0x0 mtime 2019-12-07T19:33:00.953767+0000 allocated 0 extents []) 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bf900 pos 0x0 0x0~25 to file(ino 5 size 0x0 mtime 2019-12-07T19:33:00.953767+0000 allocated 0 extents []) 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs _allocate len 0x25 from 1 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 allocate extent: 0xacd0e0000~10000/10000,10000,0 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 20 bluefs _flush_range dirty_seq = 5 (was clean) 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacd0e0000~10000 x_off 0x0 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 20 bluefs _flush_range caching tail of 0x25 and padding block with 0xfdb 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bf900 pos now 0x25 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bf900 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bf900 done in 0.000000 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 20 bluefs _fsync file metadata was dirty (5) on file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]), flushing log 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 5 len 0x57 crc 0x14583d45) 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfb80 0x4000~1000 to file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfb80 pos 0x4000 0x4000~1000 to file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x4000 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfb80 pos now 0x5000 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 done in 0.000019 2019-12-07T19:33:00.953+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 5 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e37bf900 type 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 10 bluefs rename db/000000.dbtmp -> db/IDENTITY 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 10 bluefs rename db/IDENTITY file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 10 bluefs open_for_read db/IDENTITY (random) 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d290 on file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d290 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 20 bluefs _read got 16 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 20 bluefs _read got 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d290 on file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c5e3618268) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x55c5e364cd10 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:00.955+0000 7f686bd1cbc0 20 bluefs _read got 13 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs mkdir db 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs open_for_write db.wal/000003.log 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs open_for_write mapping db.wal/000003.log vsel_hint 0 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs open_for_write h 0x55c5e37bf900 on file(ino 6 size 0x0 mtime 2019-12-07T19:33:00.956937+0000 allocated 0 extents []) 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs sync_metadata 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 6 len 0x69 crc 0x2d04f952) 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfb80 0x5000~1000 to file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfb80 pos 0x5000 0x5000~1000 to file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x5000 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfb80 pos now 0x6000 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 done in 0.000129 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 6 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs sync_metadata done in 0.000232 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs _should_compact_log current 0x6000 expected 3000 ratio 2 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs open_for_write db/OPTIONS-000004.dbtmp 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs open_for_write mapping db/OPTIONS-000004.dbtmp vsel_hint 0x1 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs open_for_write h 0x55c5e37bfcc0 on file(ino 7 size 0x0 mtime 2019-12-07T19:33:00.957243+0000 allocated 0 extents []) 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfcc0 ignoring, length 4948 < min_flush_size 524288 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs _fsync 0x55c5e37bfcc0 file(ino 7 size 0x0 mtime 2019-12-07T19:33:00.957243+0000 allocated 0 extents []) 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfcc0 0x0~1354 to file(ino 7 size 0x0 mtime 2019-12-07T19:33:00.957243+0000 allocated 0 extents []) 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfcc0 pos 0x0 0x0~1354 to file(ino 7 size 0x0 mtime 2019-12-07T19:33:00.957243+0000 allocated 0 extents []) 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs _allocate len 0x1354 from 1 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 allocate extent: 0xacd0f0000~10000/10000,10000,0 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs _flush_range dirty_seq = 7 (was clean) 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x0 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs _flush_range caching tail of 0x354 and padding block with 0xcac 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfcc0 pos now 0x1354 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfcc0 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfcc0 done in 0.000000 2019-12-07T19:33:00.956+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _fsync file metadata was dirty (7) on file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]), flushing log 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 7 len 0x60 crc 0xb01de134) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfb80 0x6000~1000 to file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfb80 pos 0x6000 0x6000~1000 to file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x6000 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfb80 pos now 0x7000 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 done in 0.000021 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 7 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfcc0 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfcc0 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e37bfcc0 type 0 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs open_for_read db/OPTIONS-000004.dbtmp (sequential) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d340 on file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d340 0x0~2000 from file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1354 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~2000 of 1:0xacd0f0000~10000 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _read left 0x2000 len 0x1354 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _read got 4948 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs rename db/OPTIONS-000004.dbtmp -> db/OPTIONS-000005 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs rename db/OPTIONS-000005 file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 4 rocksdb: DB pointer 0x55c5e424e800 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _open_fm initializing freespace 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 freelist create size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 freelist allocate 0x0~2000 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _open_fm reserved 0xbb3330000 for bluefs 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 1 freelist init 2019-12-07T19:33:00.957+0000 7f685a5c5700 4 rocksdb: [db/db_impl.cc:777] ------- DUMPING STATS ------- 2019-12-07T19:33:00.957+0000 7f685a5c5700 4 rocksdb: [db/db_impl.cc:778] ** DB Stats ** Uptime(secs): 0.0 total, 0.0 interval Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s Interval stall: 00:00:0.000 H:M:S, 0.0 percent ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_ondisk_format_super ondisk_format 3 min_compat_ondisk_format 3 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 ignoring, length 311 < min_flush_size 524288 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 ignoring, length 311 < min_flush_size 524288 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _fsync 0x55c5e37bf900 file(ino 6 size 0x0 mtime 2019-12-07T19:33:00.956937+0000 allocated 0 extents []) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 0x0~137 to file(ino 6 size 0x0 mtime 2019-12-07T19:33:00.956937+0000 allocated 0 extents []) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bf900 pos 0x0 0x0~137 to file(ino 6 size 0x0 mtime 2019-12-07T19:33:00.956937+0000 allocated 0 extents []) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _allocate len 0x137 from 0 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _allocate len 0x137 from 1 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 allocate 0x10000/10000,10000,0 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 allocate extent: 0xacd100000~10000/10000,10000,0 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _flush_range extending WAL size to 0x10000 to include allocated 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_range dirty_seq = 8 (was clean) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacd100000~10000 x_off 0x0 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_range caching tail of 0x137 and padding block with 0xec9 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_range zeroing WAL log up to 0x10000 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bf900 pos now 0x137 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bf900 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bf900 done in 0.000000 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _fsync file metadata was dirty (8) on file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]), flushing log 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 8 len 0x64 crc 0xd04f7e35) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfb80 0x7000~1000 to file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfb80 pos 0x7000 0x7000~1000 to file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x7000 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfb80 pos now 0x8000 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 done in 0.000017 2019-12-07T19:33:00.957+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.960+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 8 2019-12-07T19:33:00.960+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:00.960+0000 7f686bd1cbc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:00.960+0000 7f686bd1cbc0 10 bluefs _should_compact_log current 0x8000 expected 3000 ratio 2.66667 2019-12-07T19:33:00.960+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:00.960+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 0 meta) 2019-12-07T19:33:00.960+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _write_bdev_label path /var/lib/ceph/osd/ceph-0/block label bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 1 meta) 2019-12-07T19:33:00.960+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:00.960+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 1 meta) 2019-12-07T19:33:00.960+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _write_bdev_label path /var/lib/ceph/osd/ceph-0/block label bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 2 meta) 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _sync_bluefs_and_fm bluefs_extents at KV is now 0x[acccc0000~e6670000] 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 ignoring, length 106 < min_flush_size 524288 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 ignoring, length 106 < min_flush_size 524288 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 10 bluefs _fsync 0x55c5e37bf900 file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 0x137~6a to file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bf900 pos 0x137 0x137~6a to file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacd100000~10000 x_off 0x137 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 20 bluefs _flush_range using partial tail 0x137 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 20 bluefs _flush_range caching tail of 0x1a1 and padding block with 0xe5f 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bf900 pos now 0x1a1 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bf900 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bf900 done in 0.000000 2019-12-07T19:33:00.963+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _close_fm 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 1 freelist shutdown 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bf900 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs _truncate 0x1a1 file file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e37bf900 type 1 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs unlock_file 0x55c5e36182c0 on file(ino 2 size 0x0 mtime 2019-12-07T19:33:00.950897+0000 allocated 0 extents []) 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 1 bluefs umount 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs sync_metadata 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 9 len 0x23 crc 0x6f84983f) 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfb80 0x8000~1000 to file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfb80 pos 0x8000 0x8000~1000 to file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x8000 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfb80 pos now 0x9000 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfb80 done in 0.000018 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 9 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs sync_metadata done in 0.000096 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs _should_compact_log current 0x9000 expected 3000 ratio 3 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e37bfb80 type 0 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 20 bluefs _stop_alloc 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 1 fbmap_alloc 0x55c5e364c900 shutdown 2019-12-07T19:33:00.964+0000 7f686bd1cbc0 1 bdev(0x55c5e4298380 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:01.222+0000 7f686bd1cbc0 1 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck check (regular) 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 1 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 0 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 1 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 2 meta) 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 2 meta) 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 2 meta) 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db kv_backend = rocksdb 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 2 meta) 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db do_bluefs = 1 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db initializing bluefs 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 1 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 0 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 1 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block size 90 GiB 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 1 bluefs mount 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluefs _open_super 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 20 bluefs _init_alloc 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:01.443+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _replay 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d1e0 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_add_free done 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d1e0 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0x5d8d242d) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:00.950897+0000 allocated 0 extents []) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:00.950915+0000 allocated 0 extents []) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d1e0 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0x27b7df6c) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:00.952418+0000 allocated 0 extents []) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d1e0 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d1e0 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0x14583d45) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:00.953767+0000 allocated 0 extents []) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d1e0 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0x2d04f952) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:00.956937+0000 allocated 0 extents []) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d1e0 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0xb01de134) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:00.957243+0000 allocated 0 extents []) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d1e0 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0xd04f7e35) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d1e0 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0x6f84983f) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d1e0 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _replay 0x9000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid db82411a-1632-4be0-8c82-a647203179ea 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _replay log file size was 0x9000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 bluefs _replay done 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:01.444+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_rm_free done 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_rm_free 0xacd100000~10000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_rm_free done 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_rm_free done 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_rm_free done 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_rm_free done 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364c900 init_rm_free done 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs mount log write pos set to 0x9000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family L: 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family M: 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family P: 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options using custom Env 0x55c5e4285b40 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d550 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d550 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read got 16 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d550 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read got 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d550 on file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d550 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read got 13 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs stat db.wal/000003.log 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs stat db.wal/000003.log file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Write Ahead Log file in db.wal: 000003.log size: 417 ; 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.env: 0x55c5e4285b40 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.info_log: 0x55c5e4286120 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.write_buffer_manager: 0x55c5e434d3b0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d550 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d550 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read got 16 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d550 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read got 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d550 on file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c5e3618428) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x55c5e364ce10 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d550 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read got 13 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747181446425, "job": 1, "event": "recovery_started", "log_files": [3]} 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #3 mode 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs open_for_read db.wal/000003.log (sequential) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d6b0 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d6b0 0x0~8000 from file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1a1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0x1a1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 20 bluefs _read got 417 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747181446476, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta old nid_max 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta old blobid_max 0 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta freelist_type bitmap 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta per_pool_omap=1 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:01.445+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_csum csum_type crc32c 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 1 freelist init 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 10 freelist init size 0x1680000000 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc opening allocation metadata 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 10 freelist enumerate_next start 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:01.446+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_add_free 0x2000~167fffe000 2019-12-07T19:33:01.447+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_add_free done 2019-12-07T19:33:01.447+0000 7f686bd1cbc0 10 freelist enumerate_next end 2019-12-07T19:33:01.447+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:01.447+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:01.447+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free done 2019-12-07T19:33:01.447+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_collections 2019-12-07T19:33:01.447+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open <<>> check (regular) start 2019-12-07T19:33:01.447+0000 7f685a5c5700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:01.447+0000 7f685a5c5700 10 rocksdb: request_cache_bytes Priority: 1 Request: 0 2019-12-07T19:33:01.447+0000 7f685a5c5700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:01.447+0000 7f685a5c5700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:01.447+0000 7f685a5c5700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:01.447+0000 7f685a5c5700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:01.447+0000 7f685a5c5700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:01.447+0000 7f685a5c5700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:01.447+0000 7f685a5c5700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:01.447+0000 7f685a5c5700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:01.447+0000 7f685a5c5700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:01.447+0000 7f685a5c5700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:01.447+0000 7f685a5c5700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:01.447+0000 7f685a5c5700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0263422 2019-12-07T19:33:01.447+0000 7f685a5c5700 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:01.447+0000 7f685a5c5700 5 bluestore.MempoolThread(0x55c5e428aa08) _resize_shards cache_size: 2845357806 kv_alloc: 1073741824 kv_used: 0 meta_alloc: 1073741824 meta_used: 0 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:01.450+0000 7f686bd1cbc0 1 bluefs fsck 2019-12-07T19:33:01.450+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fffe000/0x40000000/0x1680000000, data 0x0/0x0, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:01.450+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open walking object keyspace 2019-12-07T19:33:01.452+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking shared_blobs 2019-12-07T19:33:01.452+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking pool_statfs 2019-12-07T19:33:01.452+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking for stray omap data 2019-12-07T19:33:01.452+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking deferred events 2019-12-07T19:33:01.452+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking freelist vs allocated 2019-12-07T19:33:01.453+0000 7f686bd1cbc0 10 freelist enumerate_next start 2019-12-07T19:33:01.453+0000 7f686bd1cbc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:01.498+0000 7f685a5c5700 20 bluestore.MempoolThread(0x55c5e428aa08) _resize_shards cache_size: 2845357806 kv_alloc: 1073741824 kv_used: 0 meta_alloc: 1073741824 meta_used: 0 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:01.515+0000 7f686bd1cbc0 10 freelist enumerate_next end 2019-12-07T19:33:01.516+0000 7f686bd1cbc0 2 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open 0 objects, 0 of them sharded. 2019-12-07T19:33:01.516+0000 7f686bd1cbc0 2 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open 0 extents to 0 blobs, 0 spanning, 0 shared. 2019-12-07T19:33:01.516+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open <<>> with 0 errors, 0 warnings, 0 repaired, 0 remaining in 0.069114 seconds 2019-12-07T19:33:01.516+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _flush_cache 2019-12-07T19:33:01.516+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:01.516+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:01.517+0000 7f686bd1cbc0 1 bluefs umount 2019-12-07T19:33:01.517+0000 7f686bd1cbc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:01.517+0000 7f686bd1cbc0 10 bluefs _should_compact_log current 0x9000 expected 3000 ratio 3 2019-12-07T19:33:01.517+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e37bf900 type 0 2019-12-07T19:33:01.517+0000 7f686bd1cbc0 20 bluefs _stop_alloc 2019-12-07T19:33:01.517+0000 7f686bd1cbc0 1 fbmap_alloc 0x55c5e364c900 shutdown 2019-12-07T19:33:01.517+0000 7f686bd1cbc0 1 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:01.698+0000 7f686bd1cbc0 1 fbmap_alloc 0x55c5e364cf00 shutdown 2019-12-07T19:33:01.698+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _close_fm 2019-12-07T19:33:01.698+0000 7f686bd1cbc0 1 freelist shutdown 2019-12-07T19:33:01.698+0000 7f686bd1cbc0 1 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:01.948+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:01.948+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 2 meta) 2019-12-07T19:33:01.948+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _write_bdev_label path /var/lib/ceph/osd/ceph-0/block label bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 3 meta) 2019-12-07T19:33:01.948+0000 7f686bd1cbc0 0 bluestore(/var/lib/ceph/osd/ceph-0) mkfs success 2019-12-07T19:33:01.948+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) set_cache_shards 1 2019-12-07T19:33:01.948+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _mount path /var/lib/ceph/osd/ceph-0 2019-12-07T19:33:01.948+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:01.948+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 3 meta) 2019-12-07T19:33:01.948+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck check (regular) 2019-12-07T19:33:01.948+0000 7f686bd1cbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:01.948+0000 7f686bd1cbc0 1 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:01.948+0000 7f686bd1cbc0 0 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:01.948+0000 7f686bd1cbc0 1 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:01.948+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 3 meta) 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 3 meta) 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 3 meta) 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db kv_backend = rocksdb 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 3 meta) 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db do_bluefs = 1 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db initializing bluefs 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 1 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 0 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 1 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block size 90 GiB 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 1 bluefs mount 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluefs _open_super 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 20 bluefs _init_alloc 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluefs _replay 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.949+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_add_free done 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0x5d8d242d) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:00.950897+0000 allocated 0 extents []) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:00.950915+0000 allocated 0 extents []) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0x27b7df6c) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:00.952418+0000 allocated 0 extents []) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0x14583d45) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:00.953767+0000 allocated 0 extents []) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0x2d04f952) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:00.956937+0000 allocated 0 extents []) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0xb01de134) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:00.957243+0000 allocated 0 extents []) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0xd04f7e35) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0x6f84983f) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _replay 0x9000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid db82411a-1632-4be0-8c82-a647203179ea 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _replay log file size was 0x9000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _replay done 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free done 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free 0xacd100000~10000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free done 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free done 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free done 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free done 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free done 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs mount log write pos set to 0x9000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family L: 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family M: 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family P: 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options using custom Env 0x55c5e42873c0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d970 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 16 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d970 on file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 13 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs stat db.wal/000003.log 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs stat db.wal/000003.log file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Write Ahead Log file in db.wal: 000003.log size: 417 ; 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.env: 0x55c5e42873c0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.info_log: 0x55c5e4286ba0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.write_buffer_manager: 0x55c5e434d9b0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d970 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 16 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d970 on file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c5e3618460) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x55c5e364c910 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d970 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 13 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747181951664, "job": 1, "event": "recovery_started", "log_files": [3]} 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #3 mode 0 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs open_for_read db.wal/000003.log (sequential) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d810 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d810 0x0~8000 from file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1a1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0x1a1 2019-12-07T19:33:01.950+0000 7f686bd1cbc0 20 bluefs _read got 417 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747181951704, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta old nid_max 0 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta old blobid_max 0 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta freelist_type bitmap 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta per_pool_omap=1 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_csum csum_type crc32c 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 1 freelist init 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 freelist init size 0x1680000000 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc opening allocation metadata 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 freelist enumerate_next start 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:01.951+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_add_free 0x2000~167fffe000 2019-12-07T19:33:01.952+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_add_free done 2019-12-07T19:33:01.952+0000 7f686bd1cbc0 10 freelist enumerate_next end 2019-12-07T19:33:01.952+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:01.952+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:01.952+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_rm_free done 2019-12-07T19:33:01.952+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_collections 2019-12-07T19:33:01.952+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open <<>> check (regular) start 2019-12-07T19:33:01.952+0000 7f685a3bf700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:01.952+0000 7f685a3bf700 10 rocksdb: request_cache_bytes Priority: 1 Request: 0 2019-12-07T19:33:01.952+0000 7f685a3bf700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:01.952+0000 7f685a3bf700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:01.952+0000 7f685a3bf700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:01.952+0000 7f685a3bf700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:01.952+0000 7f685a3bf700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:01.952+0000 7f685a3bf700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:01.952+0000 7f685a3bf700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:01.952+0000 7f685a3bf700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:01.952+0000 7f685a3bf700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:01.952+0000 7f685a3bf700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:01.952+0000 7f685a3bf700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:01.952+0000 7f685a3bf700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0263425 2019-12-07T19:33:01.952+0000 7f685a3bf700 5 bluestore.MempoolThread(0x55c5e428aa08) _resize_shards cache_size: 2845357735 kv_alloc: 1073741824 kv_used: 0 meta_alloc: 1073741824 meta_used: 0 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:01.954+0000 7f686bd1cbc0 1 bluefs fsck 2019-12-07T19:33:01.954+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fffe000/0x40000000/0x1680000000, data 0x0/0x0, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:01.954+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open walking object keyspace 2019-12-07T19:33:01.955+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking shared_blobs 2019-12-07T19:33:01.955+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking pool_statfs 2019-12-07T19:33:01.955+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking for stray omap data 2019-12-07T19:33:01.955+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking deferred events 2019-12-07T19:33:01.955+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking freelist vs allocated 2019-12-07T19:33:01.957+0000 7f686bd1cbc0 10 freelist enumerate_next start 2019-12-07T19:33:01.957+0000 7f686bd1cbc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:02.002+0000 7f685a3bf700 20 bluestore.MempoolThread(0x55c5e428aa08) _resize_shards cache_size: 2845357735 kv_alloc: 1073741824 kv_used: 0 meta_alloc: 1073741824 meta_used: 0 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:02.019+0000 7f686bd1cbc0 10 freelist enumerate_next end 2019-12-07T19:33:02.020+0000 7f686bd1cbc0 2 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open 0 objects, 0 of them sharded. 2019-12-07T19:33:02.020+0000 7f686bd1cbc0 2 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open 0 extents to 0 blobs, 0 spanning, 0 shared. 2019-12-07T19:33:02.020+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open <<>> with 0 errors, 0 warnings, 0 repaired, 0 remaining in 0.068297 seconds 2019-12-07T19:33:02.020+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _flush_cache 2019-12-07T19:33:02.020+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:02.020+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:02.020+0000 7f686bd1cbc0 1 bluefs umount 2019-12-07T19:33:02.020+0000 7f686bd1cbc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:02.020+0000 7f686bd1cbc0 10 bluefs _should_compact_log current 0x9000 expected 3000 ratio 3 2019-12-07T19:33:02.020+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e37bfe00 type 0 2019-12-07T19:33:02.020+0000 7f686bd1cbc0 20 bluefs _stop_alloc 2019-12-07T19:33:02.020+0000 7f686bd1cbc0 1 fbmap_alloc 0x55c5e364cf00 shutdown 2019-12-07T19:33:02.020+0000 7f686bd1cbc0 1 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:02.204+0000 7f686bd1cbc0 1 fbmap_alloc 0x55c5e364ca00 shutdown 2019-12-07T19:33:02.204+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _close_fm 2019-12-07T19:33:02.204+0000 7f686bd1cbc0 1 freelist shutdown 2019-12-07T19:33:02.204+0000 7f686bd1cbc0 1 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 1 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 0 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 1 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 3 meta) 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 3 meta) 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 3 meta) 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db kv_backend = rocksdb 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 3 meta) 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db do_bluefs = 1 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db initializing bluefs 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 1 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 0 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 1 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block size 90 GiB 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 1 bluefs mount 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluefs _open_super 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 20 bluefs _init_alloc 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:02.456+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _replay 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_add_free done 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0x5d8d242d) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:00.950897+0000 allocated 0 extents []) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:00.950915+0000 allocated 0 extents []) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0x27b7df6c) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:00.952418+0000 allocated 0 extents []) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0x14583d45) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:00.953767+0000 allocated 0 extents []) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0x2d04f952) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:00.956937+0000 allocated 0 extents []) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0xb01de134) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:00.957243+0000 allocated 0 extents []) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0xd04f7e35) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0x6f84983f) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _replay 0x9000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid db82411a-1632-4be0-8c82-a647203179ea 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _replay log file size was 0x9000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _replay done 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_rm_free done 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_rm_free 0xacd100000~10000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_rm_free done 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_rm_free done 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_rm_free done 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_rm_free done 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364ca00 init_rm_free done 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs mount log write pos set to 0x9000 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family L: 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family M: 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family P: 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options using custom Env 0x55c5e4286160 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d290 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:02.457+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read got 16 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read got 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d290 on file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read got 13 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs stat db.wal/000003.log 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs stat db.wal/000003.log file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Write Ahead Log file in db.wal: 000003.log size: 417 ; 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.env: 0x55c5e4286160 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.info_log: 0x55c5e42877a0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.write_buffer_manager: 0x55c5e42497a0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d290 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read got 16 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read got 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d290 on file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c5e36184b0) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x55c5e364cf10 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read got 13 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747182459296, "job": 1, "event": "recovery_started", "log_files": [3]} 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #3 mode 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs open_for_read db.wal/000003.log (sequential) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d1e0 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d1e0 0x0~8000 from file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1a1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0x1a1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 20 bluefs _read got 417 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747182459350, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta old nid_max 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta old blobid_max 0 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta freelist_type bitmap 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta per_pool_omap=1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_csum csum_type crc32c 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 1 freelist init 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 freelist init size 0x1680000000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:02.458+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364d000 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc opening allocation metadata 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 10 freelist enumerate_next start 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 10 freelist enumerate_next 0x2000~167fffe000 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364d000 init_add_free 0x2000~167fffe000 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364d000 init_add_free done 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 10 freelist enumerate_next end 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364d000 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364d000 init_rm_free done 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 1 bluefs umount 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 10 bluefs _should_compact_log current 0x9000 expected 3000 ratio 3 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e37bfcc0 type 0 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 20 bluefs _stop_alloc 2019-12-07T19:33:02.459+0000 7f686bd1cbc0 1 fbmap_alloc 0x55c5e364ca00 shutdown 2019-12-07T19:33:02.460+0000 7f686bd1cbc0 1 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 3 meta) 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db kv_backend = rocksdb 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 3 meta) 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db do_bluefs = 1 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db initializing bluefs 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 1 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 0 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 1 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block size 90 GiB 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 1 bluefs mount 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluefs _open_super 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _init_alloc 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluefs _replay 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_add_free done 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0x5d8d242d) 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:00.950897+0000 allocated 0 extents []) 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:00.950915+0000 allocated 0 extents []) 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:02.713+0000 7f686bd1cbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0x27b7df6c) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:00.952418+0000 allocated 0 extents []) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0x14583d45) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:00.953767+0000 allocated 0 extents []) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0x2d04f952) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:00.956937+0000 allocated 0 extents []) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0xb01de134) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:00.957243+0000 allocated 0 extents []) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0xd04f7e35) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0x6f84983f) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 4096 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _replay 0x9000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid db82411a-1632-4be0-8c82-a647203179ea 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _replay log file size was 0x9000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _replay done 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free done 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free 0xacd100000~10000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free done 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free done 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free done 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free done 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 init_rm_free done 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs mount log write pos set to 0x9000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family L: 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family M: 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family P: 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options using custom Env 0x55c5e4286960 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d290 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 16 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d290 on file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 13 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: SST files in db dir, Total Num: 0, files: 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs stat db.wal/000003.log 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs stat db.wal/000003.log file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Write Ahead Log file in db.wal: 000003.log size: 417 ; 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.env: 0x55c5e4286960 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.info_log: 0x55c5e4287620 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.write_buffer_manager: 0x55c5e4249770 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs mkdir db.wal 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs mkdir db 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs mkdir db 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs mkdir db.wal 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs dir_exists db.wal = 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs lock_file db/LOCK 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs lock_file locked file(ino 2 size 0x0 mtime 2019-12-07T19:33:00.950897+0000 allocated 0 extents []) with 0x55c5e3618448 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs stat db/CURRENT 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs stat db/CURRENT file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs stat db/IDENTITY 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs stat db/IDENTITY file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs open_for_read db/IDENTITY (random) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d290 on file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d290 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x0~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 16 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x10~2000 from file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000001 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs open_for_read db/MANIFEST-000001 (sequential) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d290 on file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs stat db/MANIFEST-000001 file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c5e36184d0) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x55c5e364d110 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d290 0x0~8000 from file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0xd 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0c0000~10000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0xd 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 13 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs mkdir db 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747182715395, "job": 1, "event": "recovery_started", "log_files": [3]} 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #3 mode 0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs open_for_read db.wal/000003.log (sequential) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d340 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d340 0x0~8000 from file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1a1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0x1a1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _read got 417 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs open_for_write db/000004.sst 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs open_for_write mapping db/000004.sst vsel_hint 0x1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs open_for_write h 0x55c5e37bfa40 on file(ino 8 size 0x0 mtime 2019-12-07T19:33:02.715439+0000 allocated 0 extents []) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfa40 ignoring, length 1383 < min_flush_size 524288 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _fsync 0x55c5e37bfa40 file(ino 8 size 0x0 mtime 2019-12-07T19:33:02.715439+0000 allocated 0 extents []) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfa40 0x0~567 to file(ino 8 size 0x0 mtime 2019-12-07T19:33:02.715439+0000 allocated 0 extents []) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfa40 pos 0x0 0x0~567 to file(ino 8 size 0x0 mtime 2019-12-07T19:33:02.715439+0000 allocated 0 extents []) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _allocate len 0x567 from 1 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 allocate extent: 0xacd110000~10000/10000,10000,0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _flush_range dirty_seq = 10 (was clean) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacd110000~10000 x_off 0x0 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _flush_range caching tail of 0x567 and padding block with 0xa99 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfa40 pos now 0x567 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfa40 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfa40 done in 0.000000 2019-12-07T19:33:02.714+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _fsync file metadata was dirty (10) on file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]), flushing log 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 10 len 0x56 crc 0x9f95ef2d) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfcc0 0x9000~1000 to file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfcc0 pos 0x9000 0x9000~1000 to file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x9000 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfcc0 pos now 0xa000 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfcc0 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfcc0 done in 0.000024 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 10 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfa40 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfa40 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e37bfa40 type 2 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs open_for_read db/000004.sst (random) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d340 on file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d340 0x0~567 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) prefetch 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd110000~10000 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _read left 0x1000 len 0x567 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _read got 1383 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _read_random h 0x55c5e361d340 0x532~35 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _read_random left 0xace 0x532~35 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _read_random got 53 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _read_random h 0x55c5e361d340 0x4dd~55 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _read_random left 0xb23 0x4dd~55 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _read_random got 85 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _read_random h 0x55c5e361d340 0x225~2b8 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _read_random left 0xddb 0x225~2b8 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _read_random got 696 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _read_random h 0x55c5e361d340 0x209~1c from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _read_random left 0xdf7 0x209~1c 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _read_random got 28 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _read_random h 0x55c5e361d340 0x1bf~4a from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _read_random left 0xe41 0x1bf~4a 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _read_random got 74 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747182715967, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1383, "table_properties": {"data_size": 447, "index_size": 28, "filter_size": 69, "raw_key_size": 314, "raw_average_key_size": 22, "raw_value_size": 127, "raw_average_value_size": 9, "num_data_blocks": 1, "num_entries": 14, "filter_policy_name": "rocksdb.BuiltinBloomFilter"}} 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 4 rocksdb: [db/version_set.cc:3036] Creating manifest 5 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs open_for_write db/MANIFEST-000005 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs open_for_write mapping db/MANIFEST-000005 vsel_hint 0x1 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs open_for_write h 0x55c5e37bfa40 on file(ino 9 size 0x0 mtime 2019-12-07T19:33:02.716009+0000 allocated 0 extents []) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfa40 ignoring, length 35 < min_flush_size 524288 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfa40 ignoring, length 44 < min_flush_size 524288 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfa40 ignoring, length 102 < min_flush_size 524288 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfa40 ignoring, length 102 < min_flush_size 524288 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _fsync 0x55c5e37bfa40 file(ino 9 size 0x0 mtime 2019-12-07T19:33:02.716009+0000 allocated 0 extents []) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfa40 0x0~66 to file(ino 9 size 0x0 mtime 2019-12-07T19:33:02.716009+0000 allocated 0 extents []) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfa40 pos 0x0 0x0~66 to file(ino 9 size 0x0 mtime 2019-12-07T19:33:02.716009+0000 allocated 0 extents []) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _allocate len 0x66 from 1 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 allocate extent: 0xacd120000~10000/10000,10000,0 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range dirty_seq = 11 (was clean) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x0 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range caching tail of 0x66 and padding block with 0xf9a 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfa40 pos now 0x66 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfa40 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfa40 done in 0.000000 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _fsync file metadata was dirty (11) on file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]), flushing log 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 11 len 0x5a crc 0xeeb8e4c4) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfcc0 0xa000~1000 to file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfcc0 pos 0xa000 0xa000~1000 to file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xa000 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfcc0 pos now 0xb000 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfcc0 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfcc0 done in 0.000017 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 11 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs open_for_write db/000005.dbtmp 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs open_for_write mapping db/000005.dbtmp vsel_hint 0x1 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs open_for_write h 0x55c5e36de000 on file(ino 10 size 0x0 mtime 2019-12-07T19:33:02.716280+0000 allocated 0 extents []) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _fsync 0x55c5e36de000 file(ino 10 size 0x0 mtime 2019-12-07T19:33:02.716280+0000 allocated 0 extents []) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e36de000 0x0~10 to file(ino 10 size 0x0 mtime 2019-12-07T19:33:02.716280+0000 allocated 0 extents []) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e36de000 pos 0x0 0x0~10 to file(ino 10 size 0x0 mtime 2019-12-07T19:33:02.716280+0000 allocated 0 extents []) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _allocate len 0x10 from 1 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 allocate extent: 0xacd130000~10000/10000,10000,0 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range dirty_seq = 12 (was clean) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x0 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range caching tail of 0x10 and padding block with 0xff0 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e36de000 pos now 0x10 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e36de000 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e36de000 done in 0.000000 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _fsync file metadata was dirty (12) on file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]), flushing log 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 12 len 0x57 crc 0x259a0ad3) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfcc0 0xb000~1000 to file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfcc0 pos 0xb000 0xb000~1000 to file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xb000 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfcc0 pos now 0xc000 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfcc0 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfcc0 done in 0.000014 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 12 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e36de000 type 0 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs rename db/000005.dbtmp -> db/CURRENT 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs rename dir db (0x55c5e4288a20) file CURRENT already exists, unlinking 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _drop_link had refs 1 on file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _drop_link destroying file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs rename db/CURRENT file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747182716490, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs open_for_write db.wal/000006.log 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs open_for_write mapping db.wal/000006.log vsel_hint 0 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs open_for_write h 0x55c5e36de000 on file(ino 11 size 0x0 mtime 2019-12-07T19:33:02.716499+0000 allocated 0 extents []) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs unlink db.wal/000003.log 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _drop_link had refs 1 on file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _drop_link destroying file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs unlink db/MANIFEST-000001 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _drop_link had refs 1 on file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _drop_link destroying file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs sync_metadata 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 13 len 0xc8 crc 0xad01a53a) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfcc0 0xc000~1000 to file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfcc0 pos 0xc000 0xc000~1000 to file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xc000 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfcc0 pos now 0xd000 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfcc0 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfcc0 done in 0.000011 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 13 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 release 0xacd0c0000~20000 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 release 0xacd100000~10000 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 release done 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs sync_metadata done in 0.000091 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs _should_compact_log current 0xd000 expected 3000 ratio 4.33333 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs open_for_write db/OPTIONS-000007.dbtmp 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 20 bluefs open_for_write mapping db/OPTIONS-000007.dbtmp vsel_hint 0x1 2019-12-07T19:33:02.715+0000 7f686bd1cbc0 10 bluefs open_for_write h 0x55c5e36de140 on file(ino 12 size 0x0 mtime 2019-12-07T19:33:02.716642+0000 allocated 0 extents []) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e36de140 ignoring, length 4949 < min_flush_size 524288 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _fsync 0x55c5e36de140 file(ino 12 size 0x0 mtime 2019-12-07T19:33:02.716642+0000 allocated 0 extents []) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e36de140 0x0~1355 to file(ino 12 size 0x0 mtime 2019-12-07T19:33:02.716642+0000 allocated 0 extents []) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e36de140 pos 0x0 0x0~1355 to file(ino 12 size 0x0 mtime 2019-12-07T19:33:02.716642+0000 allocated 0 extents []) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _allocate len 0x1355 from 1 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364cf00 allocate extent: 0xacd0c0000~10000/10000,10000,0 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _flush_range dirty_seq = 14 (was clean) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 12 size 0x1355 mtime 2019-12-07T19:33:02.716978+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacd0c0000~10000 x_off 0x0 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _flush_range caching tail of 0x355 and padding block with 0xcab 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e36de140 pos now 0x1355 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e36de140 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e36de140 done in 0.000000 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _fsync file metadata was dirty (14) on file(ino 12 size 0x1355 mtime 2019-12-07T19:33:02.716978+0000 allocated 10000 extents [1:0xacd0c0000~10000]), flushing log 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 12 size 0x1355 mtime 2019-12-07T19:33:02.716978+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 14 len 0x60 crc 0x56fcbb5d) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfcc0 0xd000~1000 to file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfcc0 pos 0xd000 0xd000~1000 to file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xd000 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfcc0 pos now 0xe000 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfcc0 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfcc0 done in 0.000014 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 14 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 12 size 0x1355 mtime 2019-12-07T19:33:02.716978+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e36de140 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e36de140 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e36de140 type 0 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs open_for_read db/OPTIONS-000007.dbtmp (sequential) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs open_for_read h 0x55c5e361d550 on file(ino 12 size 0x1355 mtime 2019-12-07T19:33:02.716978+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _read h 0x55c5e361d550 0x0~2000 from file(ino 12 size 0x1355 mtime 2019-12-07T19:33:02.716978+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1355 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _read fetching 0x0~2000 of 1:0xacd0c0000~10000 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _read left 0x2000 len 0x1355 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _read got 4949 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs rename db/OPTIONS-000007.dbtmp -> db/OPTIONS-000008 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs rename db/OPTIONS-000008 file(ino 12 size 0x1355 mtime 2019-12-07T19:33:02.716978+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 4 rocksdb: DB pointer 0x55c5e431c800 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _upgrade_super from 3, latest 3 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _upgrade_super done 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_collections 2019-12-07T19:33:02.716+0000 7f685abc0700 4 rocksdb: [db/db_impl.cc:777] ------- DUMPING STATS ------- 2019-12-07T19:33:02.716+0000 7f685abc0700 4 rocksdb: [db/db_impl.cc:778] ** DB Stats ** Uptime(secs): 0.0 total, 0.0 interval Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s Interval stall: 00:00:0.000 H:M:S, 0.0 percent ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 1/0 1.35 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Sum 1/0 1.35 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 1/0 1.35 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Sum 1/0 1.35 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.3 0.00 0.00 1 0.001 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluefs _read_random h 0x55c5e361d340 0x0~1bf from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _read_random left 0x1000 0x0~1bf 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluefs _read_random got 447 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fffe000/0x40000000/0x1680000000, data 0x0/0x0, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_start 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_replay start 2019-12-07T19:33:02.716+0000 7f685c3c3700 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread start 2019-12-07T19:33:02.716+0000 7f68613cd700 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread start 2019-12-07T19:33:02.716+0000 7f68613cd700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:02.716+0000 7f685c3c3700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach meta fresh osr 0x55c5e36de140 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_replay draining osr 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_register_zombie 0x55c5e36de140 meta 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_drain_all 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _osr_drain_all osr_set 0x55c5e36de140 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _osr_drain_all drain 0x55c5e36de140 2019-12-07T19:33:02.716+0000 7f685c3c3700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:02.716+0000 7f685c3c3700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:02.716+0000 7f68613cd700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:02.716+0000 7f68613cd700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_drain_all reaping empty zombie osr 0x55c5e36de140 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_drain_all done 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_replay completed 0 events 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluestore.OnodeSpace(0x55c5e364b5e0 in 0x55c5e4292000) clear 2019-12-07T19:33:02.716+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach meta fresh osr 0x55c5e36de140 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x55c5e364b4a0 meta 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x55c5e36de140 = 0x55c5e3668f00 seq 1 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection meta bits 0 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection meta bits 0 = 0 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x55c5e364b4a0) get_onode oid #-1:7b3f43c4:::osd_superblock:0# key 0x7f7fffffffffffffff7b3f43c4216f73'd_superblock!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x55c5e364b4a0) r -2 v.len 0 2019-12-07T19:33:02.717+0000 7f6860bcc700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:02.717+0000 7f6860bcc700 10 rocksdb: request_cache_bytes Priority: 1 Request: 944 2019-12-07T19:33:02.717+0000 7f6860bcc700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:02.717+0000 7f6860bcc700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:02.717+0000 7f6860bcc700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:02.717+0000 7f6860bcc700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~233 2019-12-07T19:33:02.717+0000 7f6860bcc700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1 2019-12-07T19:33:02.717+0000 7f6860bcc700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:02.717+0000 7f6860bcc700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:02.717+0000 7f6860bcc700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:02.717+0000 7f6860bcc700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:02.717+0000 7f6860bcc700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~233 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:02.717+0000 7f6860bcc700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:02.717+0000 7f6860bcc700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.026309 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~233 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xdcd on front/back, now 0x0~1000 2019-12-07T19:33:02.717+0000 7f6860bcc700 5 bluestore.MempoolThread(0x55c5e428aa08) _resize_shards cache_size: 2845365766 kv_alloc: 1073741824 kv_used: 944 meta_alloc: 1073741824 meta_used: 1112 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x55c5e3668f00 1 blobs 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364d000 allocate 0x1000/1000,1000,0 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 fbmap_alloc 0x55c5e364d000 allocate extent: 0x2000~1000/1000,1000,0 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x2000~1000] 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x55c5e428e310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x55c5e428ecb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x55c5e428e310 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x55c5e428ecb0 sbid 0x0)) 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore.blob(0x55c5e428e310) get_ref 0x0~233 Blob(0x55c5e428e310 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x55c5e428ecb0 sbid 0x0)) 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore.blob(0x55c5e428e310) get_ref init 0x1000, 1000 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~233: 0x0~233 Blob(0x55c5e428e310 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x55c5e428ecb0 sbid 0x0)) 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore.BufferSpace(0x55c5e428ecc8 in 0x55c5e361f340) _discard 0x0~1000 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x233 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore.extentmap(0x55c5e366b008) dirty_range mark inline shard dirty 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~233 = 0 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x55c5e3668f00 cost 8848 (2 ios * 4000 + 848 bytes) 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x55c5e3668f00 onodes 0x55c5e366af00 shared_blobs 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore.extentmap(0x55c5e366b008) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore.extentmap(0x55c5e366b008) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x55c5e3668f00 allocated 0x[2000~1000] released 0x[] 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 freelist allocate 0x2000~1000 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x55c5e3668f00 prepare 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x55c5e3668f00 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 3 meta) 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _write_bdev_label path /var/lib/ceph/osd/ceph-0/block label bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 4 meta) 2019-12-07T19:33:02.717+0000 7f6864dda700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x55c5e3668f00 aio_wait 2019-12-07T19:33:02.717+0000 7f6864dda700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x55c5e3668f00 2019-12-07T19:33:02.717+0000 7f6864dda700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x55c5e3668f00 io_done 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread new_nid_max 1025 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread new_blobid_max 10240 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs _flush 0x55c5e36de000 ignoring, length 265 < min_flush_size 524288 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x55c5e366af00 had 1 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs get_usage bdev 1 free 3860922368 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998813) bluestore 86 GiB free (0.959999), bluefs_ratio 0.0399548 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs _flush 0x55c5e36de000 ignoring, length 288 < min_flush_size 524288 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs _flush 0x55c5e36de000 ignoring, length 288 < min_flush_size 524288 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs _fsync 0x55c5e36de000 file(ino 11 size 0x0 mtime 2019-12-07T19:33:02.716499+0000 allocated 0 extents []) 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs _flush 0x55c5e36de000 0x0~120 to file(ino 11 size 0x0 mtime 2019-12-07T19:33:02.716499+0000 allocated 0 extents []) 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs _flush_range 0x55c5e36de000 pos 0x0 0x0~120 to file(ino 11 size 0x0 mtime 2019-12-07T19:33:02.716499+0000 allocated 0 extents []) 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs _allocate len 0x120 from 0 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs _allocate len 0x120 from 1 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 fbmap_alloc 0x55c5e364cf00 allocate 0x10000/10000,10000,0 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 fbmap_alloc 0x55c5e364cf00 allocate extent: 0xacd0d0000~10000/10000,10000,0 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs _flush_range extending WAL size to 0x10000 to include allocated 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluefs _flush_range dirty_seq = 15 (was clean) 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluefs _flush_range file now file(ino 11 size 0x10000 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluefs _flush_range in 1:0xacd0d0000~10000 x_off 0x0 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluefs _flush_range caching tail of 0x120 and padding block with 0xee0 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluefs _flush_range zeroing WAL log up to 0x10000 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 4 meta) 2019-12-07T19:33:02.717+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _write_bdev_label path /var/lib/ceph/osd/ceph-0/block label bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 5 meta) 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluefs _flush_range h 0x55c5e36de000 pos now 0x120 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs wait_for_aio 0x55c5e36de000 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs wait_for_aio 0x55c5e36de000 done in 0.000000 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluefs flush_bdev 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluefs _fsync file metadata was dirty (15) on file(ino 11 size 0x10000 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]), flushing log 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluefs _flush_and_sync_log op_file_update file(ino 11 size 0x10000 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs _flush_and_sync_log txn(seq 15 len 0x64 crc 0xc7ef94bb) 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs _flush 0x55c5e37bfcc0 0xe000~1000 to file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs _flush_range 0x55c5e37bfcc0 pos 0xe000 0xe000~1000 to file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluefs _flush_range file now file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xe000 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluefs _flush_range h 0x55c5e37bfcc0 pos now 0xf000 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs wait_for_aio 0x55c5e37bfcc0 2019-12-07T19:33:02.717+0000 7f685c3c3700 10 bluefs wait_for_aio 0x55c5e37bfcc0 done in 0.000016 2019-12-07T19:33:02.717+0000 7f685c3c3700 20 bluefs flush_bdev 2019-12-07T19:33:02.718+0000 7f685c3c3700 20 bluefs _flush_and_sync_log log_seq_stable 15 2019-12-07T19:33:02.718+0000 7f685c3c3700 20 bluefs _flush_and_sync_log cleaned file file(ino 11 size 0x10000 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.718+0000 7f685c3c3700 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:02.718+0000 7f685c3c3700 10 bluefs _should_compact_log current 0xf000 expected 3000 ratio 5 2019-12-07T19:33:02.718+0000 7f685c3c3700 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread nid_max now 1025 2019-12-07T19:33:02.718+0000 7f685c3c3700 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread blobid_max now 10240 2019-12-07T19:33:02.718+0000 7f68613cd700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:02.718+0000 7f68613cd700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x55c5e3668f00> 2019-12-07T19:33:02.718+0000 7f68613cd700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:02.718+0000 7f685c3c3700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000374310s (0.000007671s flush + 0.000366639s kv commit) 2019-12-07T19:33:02.718+0000 7f68613cd700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x55c5e3668f00 kv_submitted 2019-12-07T19:33:02.718+0000 7f68613cd700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x55c5e3668f00 2019-12-07T19:33:02.718+0000 7f685c3c3700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:02.718+0000 7f68613cd700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x55c5e3668f00 finishing 2019-12-07T19:33:02.718+0000 7f68613cd700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x55c5e3668f00 onodes 0x55c5e366af00 2019-12-07T19:33:02.718+0000 7f68613cd700 20 bluestore.BufferSpace(0x55c5e428ecc8 in 0x55c5e361f340) _finish_write discard buffer(0x55c5e3643cb0 space 0x55c5e428ecc8 0x0~1000 writing nocache) 2019-12-07T19:33:02.718+0000 7f68613cd700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x55c5e3668f00 done 2019-12-07T19:33:02.718+0000 7f68613cd700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x55c5e36de140 q now empty 2019-12-07T19:33:02.718+0000 7f68613cd700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x55c5e3668f00 [] 2019-12-07T19:33:02.718+0000 7f68613cd700 10 fbmap_alloc 0x55c5e364d000 release done 2019-12-07T19:33:02.718+0000 7f68613cd700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:02.718+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:02.718+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 5 meta) 2019-12-07T19:33:02.718+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _write_bdev_label path /var/lib/ceph/osd/ceph-0/block label bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 6 meta) 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 6 meta) 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _write_bdev_label path /var/lib/ceph/osd/ceph-0/block label bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) umount 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_drain_all 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _osr_drain_all osr_set 0x55c5e36de140 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _osr_drain_all drain 0x55c5e36de140 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_drain_all done 2019-12-07T19:33:02.723+0000 7f685c3c3700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:02.723+0000 7f685c3c3700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:02.723+0000 7f68613cd700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:02.723+0000 7f68613cd700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) umount stopping kv thread 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_stop 2019-12-07T19:33:02.723+0000 7f685c3c3700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:02.723+0000 7f68613cd700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:02.723+0000 7f685c3c3700 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread finish 2019-12-07T19:33:02.723+0000 7f68613cd700 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread finish 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_stop stopping finishers 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_stop stopped 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _flush_cache 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 20 bluestore.sharedblob(0x55c5e428ecb0) put 0x55c5e428ecb0 removing self from set 0x55c5e364b570 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 20 bluestore.BufferSpace(0x55c5e428ecc8 in 0x55c5e361f340) _clear 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluestore.OnodeSpace(0x55c5e364b5e0 in 0x55c5e4292000) clear 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) umount closing 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs readdir db 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs readdir db.wal 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e36de000 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e36de000 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e36de000 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e36de000 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _truncate 0x120 file file(ino 11 size 0x10000 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e36de000 type 1 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfa40 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfa40 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfa40 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _truncate 0x66 file file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e37bfa40 type 0 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs unlock_file 0x55c5e3618448 on file(ino 2 size 0x0 mtime 2019-12-07T19:33:00.950897+0000 allocated 0 extents []) 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 1 bluefs umount 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs sync_metadata 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _flush_and_sync_log txn(seq 16 len 0x23 crc 0x81791f6f) 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _flush 0x55c5e37bfcc0 0xf000~1000 to file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _flush_range 0x55c5e37bfcc0 pos 0xf000 0xf000~1000 to file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 20 bluefs _flush_range file now file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0xf000 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 20 bluefs _flush_range h 0x55c5e37bfcc0 pos now 0x10000 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfcc0 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs wait_for_aio 0x55c5e37bfcc0 done in 0.000019 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 20 bluefs flush_bdev 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 20 bluefs _flush_and_sync_log log_seq_stable 16 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs sync_metadata done in 0.000094 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _should_compact_log current 0x10000 expected 3000 ratio 5.33333 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 10 bluefs _close_writer 0x55c5e37bfcc0 type 0 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 20 bluefs _stop_alloc 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 1 fbmap_alloc 0x55c5e364cf00 shutdown 2019-12-07T19:33:02.723+0000 7f686bd1cbc0 1 bdev(0x55c5e4298e00 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:02.980+0000 7f686bd1cbc0 1 fbmap_alloc 0x55c5e364d000 shutdown 2019-12-07T19:33:02.980+0000 7f686bd1cbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _close_fm 2019-12-07T19:33:02.980+0000 7f686bd1cbc0 1 freelist shutdown 2019-12-07T19:33:02.980+0000 7f686bd1cbc0 1 bdev(0x55c5e4298000 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:03.222+0000 7f686bd1cbc0 0 created object store /var/lib/ceph/osd/ceph-0 for osd.0 fsid ad52b421-bda1-4750-8dc4-c9e4d1297c01 2019-12-07T19:33:34.991+0000 7fe2ed47bbc0 0 ceph version 15.0.0-8038-g503c226 (503c226f594b239359b549e9328eeb7369e666df) octopus (dev), process ceph-osd, pid 37079 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 5 object store type is bluestore 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) set_cache_shards 1 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e000 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 0 bdev(0x558e80f1e000 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e000 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e380 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 0 bdev(0x558e80f1e380 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e380 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:34.993+0000 7fe2ed47bbc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block size 90 GiB 2019-12-07T19:33:34.994+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e380 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:35.253+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e000 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:35.504+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:35.504+0000 7fe2ed47bbc0 1 objectstore numa_node 0 2019-12-07T19:33:35.505+0000 7fe2ed47bbc0 0 starting osd.0 osd_data /var/lib/ceph/osd/ceph-0 /var/lib/ceph/osd/ceph-0/journal 2019-12-07T19:33:35.505+0000 7fe2ed47bbc0 -1 Falling back to public interface 2019-12-07T19:33:35.506+0000 7fe2ed47bbc0 1 -- [v2:0.0.0.0:6818/37079,v1:0.0.0.0:6820/37079] _finish_bind bind my_addrs is [v2:0.0.0.0:6818/37079,v1:0.0.0.0:6820/37079] 2019-12-07T19:33:35.507+0000 7fe2ed47bbc0 1 -- [v2:0.0.0.0:6822/37079,v1:0.0.0.0:6824/37079] _finish_bind bind my_addrs is [v2:0.0.0.0:6822/37079,v1:0.0.0.0:6824/37079] 2019-12-07T19:33:35.508+0000 7fe2ed47bbc0 1 -- [v2:0.0.0.0:6826/37079,v1:0.0.0.0:6828/37079] _finish_bind bind my_addrs is [v2:0.0.0.0:6826/37079,v1:0.0.0.0:6828/37079] 2019-12-07T19:33:35.509+0000 7fe2ed47bbc0 1 -- [v2:0.0.0.0:6830/37079,v1:0.0.0.0:6832/37079] _finish_bind bind my_addrs is [v2:0.0.0.0:6830/37079,v1:0.0.0.0:6832/37079] 2019-12-07T19:33:35.515+0000 7fe2ed47bbc0 0 load: jerasure load: lrc load: isa 2019-12-07T19:33:35.515+0000 7fe2ed47bbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:35.515+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e000 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:35.515+0000 7fe2ed47bbc0 0 bdev(0x558e80f1e000 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:35.515+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e000 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:35.515+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:35.515+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:35.515+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:35.515+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e000 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 0 osd.0:0.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 0 osd.0:1.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 0 osd.0:2.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 0 osd.0:3.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 0 osd.0:4.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 0 osd.0:5.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 0 osd.0:6.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 0 osd.0:7.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=64) 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 1 -- [v2:0.0.0.0:6818/37079,v1:0.0.0.0:6820/37079] start start 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 1 -- start start 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 1 -- start start 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 1 -- [v2:0.0.0.0:6826/37079,v1:0.0.0.0:6828/37079] start start 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 1 -- [v2:0.0.0.0:6830/37079,v1:0.0.0.0:6832/37079] start start 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 1 -- [v2:0.0.0.0:6822/37079,v1:0.0.0.0:6824/37079] start start 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 1 -- start start 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 2 osd.0 0 init /var/lib/ceph/osd/ceph-0 (looks like ssd) 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 2 osd.0 0 journal /var/lib/ceph/osd/ceph-0/journal 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) set_cache_shards 32 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _mount path /var/lib/ceph/osd/ceph-0 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck check (regular) 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e000 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 0 bdev(0x558e80f1e000 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e000 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:35.770+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db kv_backend = rocksdb 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db do_bluefs = 1 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db initializing bluefs 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e380 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 0 bdev(0x558e80f1e380 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e380 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block size 90 GiB 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 1 bluefs mount 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluefs _open_super 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 20 bluefs _init_alloc 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluefs _replay 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.771+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_add_free done 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0x5d8d242d) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:00.950897+0000 allocated 0 extents []) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:00.950915+0000 allocated 0 extents []) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0x27b7df6c) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:00.952418+0000 allocated 0 extents []) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0x14583d45) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:00.953767+0000 allocated 0 extents []) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0x2d04f952) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:00.956937+0000 allocated 0 extents []) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0xb01de134) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:00.957243+0000 allocated 0 extents []) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0xd04f7e35) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0x6f84983f) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _replay 0x9000: txn(seq 10 len 0x56 crc 0x9f95ef2d) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x0 mtime 2019-12-07T19:33:02.715439+0000 allocated 0 extents []) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x9000: op_dir_link db/000004.sst to 8 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0xa000~1000 from file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read left 0xf6000 len 0x1000 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 10 bluefs _replay 0xa000: txn(seq 11 len 0x5a crc 0xeeb8e4c4) 2019-12-07T19:33:35.772+0000 7fe2ed47bbc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x0 mtime 2019-12-07T19:33:02.716009+0000 allocated 0 extents []) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xa000: op_dir_link db/MANIFEST-000005 to 9 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0xb000~1000 from file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read left 0xf5000 len 0x1000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _replay 0xb000: txn(seq 12 len 0x57 crc 0x259a0ad3) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x0 mtime 2019-12-07T19:33:02.716280+0000 allocated 0 extents []) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xb000: op_dir_link db/000005.dbtmp to 10 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0xc000~1000 from file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read left 0xf4000 len 0x1000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _replay 0xc000: txn(seq 13 len 0xc8 crc 0xad01a53a) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_unlink db/CURRENT 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_file_remove 4 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_link db/CURRENT to 10 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_unlink db/000005.dbtmp 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_file_update file(ino 11 size 0x0 mtime 2019-12-07T19:33:02.716499+0000 allocated 0 extents []) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_link db.wal/000006.log to 11 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_unlink db.wal/000003.log 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_file_remove 6 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_unlink db/MANIFEST-000001 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_file_remove 3 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0xd000~1000 from file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read left 0xf3000 len 0x1000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _replay 0xd000: txn(seq 14 len 0x60 crc 0x56fcbb5d) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x0 mtime 2019-12-07T19:33:02.716642+0000 allocated 0 extents []) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xd000: op_dir_link db/OPTIONS-000007.dbtmp to 12 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x1355 mtime 2019-12-07T19:33:02.716978+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0xe000~1000 from file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read left 0xf2000 len 0x1000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _replay 0xe000: txn(seq 15 len 0x64 crc 0xc7ef94bb) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xe000: op_dir_link db/OPTIONS-000008 to 12 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xe000: op_dir_unlink db/OPTIONS-000007.dbtmp 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xe000: op_file_update file(ino 11 size 0x10000 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0xf000~1000 from file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read left 0xf1000 len 0x1000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _replay 0xf000: txn(seq 16 len 0x23 crc 0x81791f6f) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _replay 0xf000: op_file_update file(ino 11 size 0x120 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x10000~1000 from file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read left 0xf0000 len 0x1000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _replay 0x10000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid db82411a-1632-4be0-8c82-a647203179ea 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _replay log file size was 0x10000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _replay done 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free done 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free 0xacd130000~10000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free done 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free 0xacd120000~10000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free done 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free 0xacd110000~10000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free done 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free done 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free done 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free done 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80232900 init_rm_free done 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs mount log write pos set to 0x10000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs readdir db.wal 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family L: 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family M: 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family P: 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 rocksdb: load_rocksdb_options using custom Env 0x558e80e6b280 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80202000 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read got 16 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read got 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80202000 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read got 102 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs readdir db 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs readdir db 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 102 Bytes 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: SST files in db dir, Total Num: 1, files: 000004.sst 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs readdir db.wal 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs stat db.wal/000006.log 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs stat db.wal/000006.log file(ino 11 size 0x120 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Write Ahead Log file in db.wal: 000006.log size: 288 ; 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.env: 0x558e80e6b280 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.info_log: 0x558e80e6cb40 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.write_buffer_manager: 0x558e81001080 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80202000 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read got 16 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read got 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000005 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80202000 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558e801fe620) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x558e80233010 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:35.773+0000 7fe2ed47bbc0 20 bluefs _read got 102 2019-12-07T19:33:35.773+0000 7fe2dcc96700 10 bluefs open_for_read db/000004.sst (random) 2019-12-07T19:33:35.773+0000 7fe2dcc96700 10 bluefs open_for_read h 0x558e80203810 on file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.773+0000 7fe2dcc96700 10 bluefs _read h 0x558e80203810 0x0~567 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) prefetch 2019-12-07T19:33:35.773+0000 7fe2dcc96700 20 bluefs _read fetching 0x0~1000 of 1:0xacd110000~10000 2019-12-07T19:33:35.773+0000 7fe2dcc96700 20 bluefs _read left 0x1000 len 0x567 2019-12-07T19:33:35.773+0000 7fe2dcc96700 20 bluefs _read got 1383 2019-12-07T19:33:35.773+0000 7fe2dcc96700 10 bluefs _read_random h 0x558e80203810 0x532~35 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.773+0000 7fe2dcc96700 20 bluefs _read_random left 0xace 0x532~35 2019-12-07T19:33:35.773+0000 7fe2dcc96700 20 bluefs _read_random got 53 2019-12-07T19:33:35.774+0000 7fe2dcc96700 10 bluefs _read_random h 0x558e80203810 0x4dd~55 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.774+0000 7fe2dcc96700 20 bluefs _read_random left 0xb23 0x4dd~55 2019-12-07T19:33:35.774+0000 7fe2dcc96700 20 bluefs _read_random got 85 2019-12-07T19:33:35.774+0000 7fe2dcc96700 10 bluefs _read_random h 0x558e80203810 0x225~2b8 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.774+0000 7fe2dcc96700 20 bluefs _read_random left 0xddb 0x225~2b8 2019-12-07T19:33:35.774+0000 7fe2dcc96700 20 bluefs _read_random got 696 2019-12-07T19:33:35.774+0000 7fe2dcc96700 10 bluefs _read_random h 0x558e80203810 0x209~1c from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.774+0000 7fe2dcc96700 20 bluefs _read_random left 0xdf7 0x209~1c 2019-12-07T19:33:35.774+0000 7fe2dcc96700 20 bluefs _read_random got 28 2019-12-07T19:33:35.774+0000 7fe2dcc96700 10 bluefs _read_random h 0x558e80203810 0x1bf~4a from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.774+0000 7fe2dcc96700 20 bluefs _read_random left 0xe41 0x1bf~4a 2019-12-07T19:33:35.774+0000 7fe2dcc96700 20 bluefs _read_random got 74 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 14, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 4 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluefs readdir db.wal 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747215776185, "job": 1, "event": "recovery_started", "log_files": [6]} 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #6 mode 0 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluefs open_for_read db.wal/000006.log (sequential) 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e802033f0 on file(ino 11 size 0x120 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e802033f0 0x0~8000 from file(ino 11 size 0x120 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x120 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x120 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 20 bluefs _read got 288 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747215776228, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluefs readdir db 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta old nid_max 1025 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta old blobid_max 10240 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluefs _read_random h 0x558e80203810 0x0~1bf from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 20 bluefs _read_random left 0x1000 0x0~1bf 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 20 bluefs _read_random got 447 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta freelist_type bitmap 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta per_pool_omap=1 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_csum csum_type crc32c 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 1 freelist init 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 freelist init size 0x1680000000 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:35.774+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:35.776+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc opening allocation metadata 2019-12-07T19:33:35.776+0000 7fe2ed47bbc0 10 freelist enumerate_next start 2019-12-07T19:33:35.776+0000 7fe2ed47bbc0 10 freelist enumerate_next 0x3000~167fffd000 2019-12-07T19:33:35.776+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_add_free 0x3000~167fffd000 2019-12-07T19:33:35.777+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_add_free done 2019-12-07T19:33:35.777+0000 7fe2ed47bbc0 10 freelist enumerate_next end 2019-12-07T19:33:35.777+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:35.777+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:35.777+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:35.777+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_collections 2019-12-07T19:33:35.777+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _open_collections opened meta 0x558e80230f00 cnode(bits 0) 2019-12-07T19:33:35.777+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach meta fresh osr 0x558e803a5cc0 2019-12-07T19:33:35.777+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open <<>> check (regular) start 2019-12-07T19:33:35.777+0000 7fe2d5c88700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:35.777+0000 7fe2d5c88700 10 rocksdb: request_cache_bytes Priority: 1 Request: 944 2019-12-07T19:33:35.777+0000 7fe2d5c88700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:35.777+0000 7fe2d5c88700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:35.777+0000 7fe2d5c88700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:35.777+0000 7fe2d5c88700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:35.777+0000 7fe2d5c88700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:35.777+0000 7fe2d5c88700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:35.777+0000 7fe2d5c88700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:35.777+0000 7fe2d5c88700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:35.777+0000 7fe2d5c88700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:35.777+0000 7fe2d5c88700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:35.777+0000 7fe2d5c88700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:35.777+0000 7fe2d5c88700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0263843 2019-12-07T19:33:35.777+0000 7fe2d5c88700 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:35.777+0000 7fe2d5c88700 5 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845346829 kv_alloc: 1073741824 kv_used: 944 meta_alloc: 1073741824 meta_used: 128 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:35.782+0000 7fe2ed47bbc0 1 bluefs fsck 2019-12-07T19:33:35.782+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x233/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:35.782+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open walking object keyspace 2019-12-07T19:33:35.784+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_check_objects collection meta cnode(bits 0) 2019-12-07T19:33:35.784+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) fsck_check_objects_shallow #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:35.784+0000 7fe2ed47bbc0 20 bluestore.blob(0x558e80f05f10) get_ref 0x0~233 Blob(0x558e80f05f10 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81078310 sbid 0x0)) 2019-12-07T19:33:35.784+0000 7fe2ed47bbc0 20 bluestore.blob(0x558e80f05f10) get_ref init 0x1000, 1000 2019-12-07T19:33:35.784+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) fsck_check_objects_shallow 0x0~233: 0x0~233 Blob(0x558e80f05f10 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x558e81078310 sbid 0x0)) 2019-12-07T19:33:35.784+0000 7fe2ed47bbc0 20 bluestore.sharedblob(0x558e81078310) put 0x558e81078310 removing self from set 0x558e80230fd0 2019-12-07T19:33:35.784+0000 7fe2ed47bbc0 20 bluestore.BufferSpace(0x558e81078328 in 0x558e80219340) _clear 2019-12-07T19:33:35.784+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking shared_blobs 2019-12-07T19:33:35.784+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking pool_statfs 2019-12-07T19:33:35.784+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking for stray omap data 2019-12-07T19:33:35.784+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking deferred events 2019-12-07T19:33:35.784+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open checking freelist vs allocated 2019-12-07T19:33:35.786+0000 7fe2ed47bbc0 10 freelist enumerate_next start 2019-12-07T19:33:35.786+0000 7fe2ed47bbc0 10 freelist enumerate_next 0x3000~167fffd000 2019-12-07T19:33:35.828+0000 7fe2d5c88700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845346829 kv_alloc: 1073741824 kv_used: 944 meta_alloc: 1073741824 meta_used: 128 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:35.878+0000 7fe2d5c88700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845346829 kv_alloc: 1073741824 kv_used: 944 meta_alloc: 1073741824 meta_used: 128 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:35.913+0000 7fe2ed47bbc0 10 freelist enumerate_next end 2019-12-07T19:33:35.915+0000 7fe2ed47bbc0 2 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open 1 objects, 0 of them sharded. 2019-12-07T19:33:35.915+0000 7fe2ed47bbc0 2 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open 1 extents to 1 blobs, 0 spanning, 0 shared. 2019-12-07T19:33:35.915+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _fsck_on_open <<>> with 0 errors, 0 warnings, 0 repaired, 0 remaining in 0.137617 seconds 2019-12-07T19:33:35.915+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _flush_cache 2019-12-07T19:33:35.915+0000 7fe2ed47bbc0 10 bluestore.OnodeSpace(0x558e80231040 in 0x558e8026a000) clear 2019-12-07T19:33:35.915+0000 7fe2ed47bbc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:35.915+0000 7fe2ed47bbc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:35.915+0000 7fe2ed47bbc0 1 bluefs umount 2019-12-07T19:33:35.915+0000 7fe2ed47bbc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:35.915+0000 7fe2ed47bbc0 10 bluefs _should_compact_log current 0x10000 expected 3000 ratio 5.33333 2019-12-07T19:33:35.915+0000 7fe2ed47bbc0 10 bluefs _close_writer 0x558e803a5b80 type 0 2019-12-07T19:33:35.915+0000 7fe2ed47bbc0 20 bluefs _stop_alloc 2019-12-07T19:33:35.915+0000 7fe2ed47bbc0 1 fbmap_alloc 0x558e80232900 shutdown 2019-12-07T19:33:35.915+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e380 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:36.027+0000 7fe2ed47bbc0 1 fbmap_alloc 0x558e80233100 shutdown 2019-12-07T19:33:36.027+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _close_fm 2019-12-07T19:33:36.027+0000 7fe2ed47bbc0 1 freelist shutdown 2019-12-07T19:33:36.027+0000 7fe2ed47bbc0 1 bdev(0x558e80f1e000 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 1 bdev(0x558e80f1ee00 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 bdev(0x558e80f1ee00 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 1 bdev(0x558e80f1ee00 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db kv_backend = rocksdb 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db do_bluefs = 1 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db initializing bluefs 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 1 bdev(0x558e80f1ea80 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 bdev(0x558e80f1ea80 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 1 bdev(0x558e80f1ea80 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block size 90 GiB 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 1 bluefs mount 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _open_super 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _init_alloc 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_add_free done 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0x5d8d242d) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:00.950897+0000 allocated 0 extents []) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:00.950915+0000 allocated 0 extents []) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0x27b7df6c) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:00.952418+0000 allocated 0 extents []) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0x14583d45) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:00.953767+0000 allocated 0 extents []) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0x2d04f952) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:00.956937+0000 allocated 0 extents []) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0xb01de134) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:00.957243+0000 allocated 0 extents []) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0xd04f7e35) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0x6f84983f) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0x9000: txn(seq 10 len 0x56 crc 0x9f95ef2d) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x0 mtime 2019-12-07T19:33:02.715439+0000 allocated 0 extents []) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x9000: op_dir_link db/000004.sst to 8 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0xa000~1000 from file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xf6000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0xa000: txn(seq 11 len 0x5a crc 0xeeb8e4c4) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x0 mtime 2019-12-07T19:33:02.716009+0000 allocated 0 extents []) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xa000: op_dir_link db/MANIFEST-000005 to 9 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0xb000~1000 from file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xf5000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0xb000: txn(seq 12 len 0x57 crc 0x259a0ad3) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x0 mtime 2019-12-07T19:33:02.716280+0000 allocated 0 extents []) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xb000: op_dir_link db/000005.dbtmp to 10 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0xc000~1000 from file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xf4000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0xc000: txn(seq 13 len 0xc8 crc 0xad01a53a) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_unlink db/CURRENT 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_file_remove 4 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_link db/CURRENT to 10 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_unlink db/000005.dbtmp 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_file_update file(ino 11 size 0x0 mtime 2019-12-07T19:33:02.716499+0000 allocated 0 extents []) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_link db.wal/000006.log to 11 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_unlink db.wal/000003.log 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_file_remove 6 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_unlink db/MANIFEST-000001 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_file_remove 3 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0xd000~1000 from file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xf3000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0xd000: txn(seq 14 len 0x60 crc 0x56fcbb5d) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x0 mtime 2019-12-07T19:33:02.716642+0000 allocated 0 extents []) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xd000: op_dir_link db/OPTIONS-000007.dbtmp to 12 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x1355 mtime 2019-12-07T19:33:02.716978+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0xe000~1000 from file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xf2000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0xe000: txn(seq 15 len 0x64 crc 0xc7ef94bb) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xe000: op_dir_link db/OPTIONS-000008 to 12 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xe000: op_dir_unlink db/OPTIONS-000007.dbtmp 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xe000: op_file_update file(ino 11 size 0x10000 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0xf000~1000 from file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xf1000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0xf000: txn(seq 16 len 0x23 crc 0x81791f6f) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _replay 0xf000: op_file_update file(ino 11 size 0x120 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80202000 0x10000~1000 from file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0xf0000 len 0x1000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay 0x10000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid db82411a-1632-4be0-8c82-a647203179ea 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay log file size was 0x10000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _replay done 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacd130000~10000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacd120000~10000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacd110000~10000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs mount log write pos set to 0x10000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family L: 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family M: 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family P: 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 rocksdb: load_rocksdb_options using custom Env 0x558e80e6d320 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80203550 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203550 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 16 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203550 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 0 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80203550 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203550 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 20 bluefs _read got 102 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:36.278+0000 7fe2ed47bbc0 10 bluefs readdir db 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs readdir db 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 102 Bytes 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: SST files in db dir, Total Num: 1, files: 000004.sst 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs stat db.wal/000006.log 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs stat db.wal/000006.log file(ino 11 size 0x120 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Write Ahead Log file in db.wal: 000006.log size: 288 ; 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.env: 0x558e80e6d320 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.info_log: 0x558e80e6cc40 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.write_buffer_manager: 0x558e81001da0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80203550 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203550 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read got 16 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203550 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read got 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000005 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80203550 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558e801fe570) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x558e80233210 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203550 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read got 102 2019-12-07T19:33:36.279+0000 7fe2dca90700 10 bluefs open_for_read db/000004.sst (random) 2019-12-07T19:33:36.279+0000 7fe2dca90700 10 bluefs open_for_read h 0x558e80203340 on file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.279+0000 7fe2dca90700 10 bluefs _read h 0x558e80203340 0x0~567 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) prefetch 2019-12-07T19:33:36.279+0000 7fe2dca90700 20 bluefs _read fetching 0x0~1000 of 1:0xacd110000~10000 2019-12-07T19:33:36.279+0000 7fe2dca90700 20 bluefs _read left 0x1000 len 0x567 2019-12-07T19:33:36.279+0000 7fe2dca90700 20 bluefs _read got 1383 2019-12-07T19:33:36.279+0000 7fe2dca90700 10 bluefs _read_random h 0x558e80203340 0x532~35 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.279+0000 7fe2dca90700 20 bluefs _read_random left 0xace 0x532~35 2019-12-07T19:33:36.279+0000 7fe2dca90700 20 bluefs _read_random got 53 2019-12-07T19:33:36.279+0000 7fe2dca90700 10 bluefs _read_random h 0x558e80203340 0x4dd~55 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.279+0000 7fe2dca90700 20 bluefs _read_random left 0xb23 0x4dd~55 2019-12-07T19:33:36.279+0000 7fe2dca90700 20 bluefs _read_random got 85 2019-12-07T19:33:36.279+0000 7fe2dca90700 10 bluefs _read_random h 0x558e80203340 0x225~2b8 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.279+0000 7fe2dca90700 20 bluefs _read_random left 0xddb 0x225~2b8 2019-12-07T19:33:36.279+0000 7fe2dca90700 20 bluefs _read_random got 696 2019-12-07T19:33:36.279+0000 7fe2dca90700 10 bluefs _read_random h 0x558e80203340 0x209~1c from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.279+0000 7fe2dca90700 20 bluefs _read_random left 0xdf7 0x209~1c 2019-12-07T19:33:36.279+0000 7fe2dca90700 20 bluefs _read_random got 28 2019-12-07T19:33:36.279+0000 7fe2dca90700 10 bluefs _read_random h 0x558e80203340 0x1bf~4a from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.279+0000 7fe2dca90700 20 bluefs _read_random left 0xe41 0x1bf~4a 2019-12-07T19:33:36.279+0000 7fe2dca90700 20 bluefs _read_random got 74 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 14, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 4 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216281244, "job": 1, "event": "recovery_started", "log_files": [6]} 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #6 mode 0 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs open_for_read db.wal/000006.log (sequential) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e802036b0 on file(ino 11 size 0x120 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e802036b0 0x0~8000 from file(ino 11 size 0x120 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x120 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x120 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read got 288 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216281277, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs readdir db 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta old nid_max 1025 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta old blobid_max 10240 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs _read_random h 0x558e80203340 0x0~1bf from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read_random left 0x1000 0x0~1bf 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 20 bluefs _read_random got 447 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta freelist_type bitmap 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta ondisk_format 3 compat_ondisk_format 3 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta min_alloc_size 0x1000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_super_meta per_pool_omap=1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_statfs per-pool statfs is enabled 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_sizes min_alloc_size 0x1000 order 12 max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_throttle_params throttle_cost_per_io 4000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_csum csum_type crc32c 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_compression mode aggressive alg snappy min_blob 8192 max_blob 65536 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_blob_size max_blob_size 0x10000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 1 freelist init 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 freelist init blocks 0x1680000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 freelist init blocks_per_key 0x80 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 freelist init bytes_per_block 0x1000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 freelist init size 0x1680000000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 freelist init size 0x1680000000 bytes_per_block 0x1000 blocks 0x1680000 blocks_per_key 0x80 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluefs get_block_extents bdev 1 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc bluefs extents 0x[acccc0000~e6670000] 2019-12-07T19:33:36.279+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233300 BitmapAllocator 0x1680000000/1000 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc opening allocation metadata 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 10 freelist enumerate_next start 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 10 freelist enumerate_next 0x3000~167fffd000 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233300 init_add_free 0x3000~167fffd000 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233300 init_add_free done 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 10 freelist enumerate_next end 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_alloc loaded 90 GiB in 1 extents 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233300 init_rm_free 0xacccc0000~e6670000 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233300 init_rm_free done 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 1 bluefs umount 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 10 bluefs _should_compact_log current 0x10000 expected 3000 ratio 5.33333 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 10 bluefs _close_writer 0x558e803a5a40 type 0 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 20 bluefs _stop_alloc 2019-12-07T19:33:36.280+0000 7fe2ed47bbc0 1 fbmap_alloc 0x558e80233100 shutdown 2019-12-07T19:33:36.281+0000 7fe2ed47bbc0 1 bdev(0x558e80f1ea80 /var/lib/ceph/osd/ceph-0/block) close 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db kv_backend = rocksdb 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db do_bluefs = 1 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_db initializing bluefs 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 1 bdev create path /var/lib/ceph/osd/ceph-0/block type kernel 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 1 bdev(0x558e80f1ea80 /var/lib/ceph/osd/ceph-0/block) open path /var/lib/ceph/osd/ceph-0/block 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 0 bdev(0x558e80f1ea80 /var/lib/ceph/osd/ceph-0/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-0/block failed: (22) Invalid argument 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 1 bdev(0x558e80f1ea80 /var/lib/ceph/osd/ceph-0/block) open size 96636764160 (0x1680000000, 90 GiB) block_size 4096 (4 KiB) non-rotational discard supported 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0/block size 90 GiB 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 1 bluefs mount 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 bluefs _open_super 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 bluefs _open_super superblock 1 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 20 bluefs _init_alloc 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x1680000000 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 BitmapAllocator 0x1680000000/10000 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 bluefs _replay 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.538+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~100000 of 1:0xacccc0000~400000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0x100000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0x0: txn(seq 1 len 0x13 crc 0xfc77ba56) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x0: op_init 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x0: op_alloc_add 1:0xacccc0000~e6670000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_add_free 0xacccc0000~e6670000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_add_free done 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x1000~1000 from file(ino 1 size 0x1000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xff000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0x1000: txn(seq 2 len 0xa5 crc 0x5d8d242d) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_create db 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_create db.slow 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_create db.wal 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_file_update file(ino 2 size 0x0 mtime 2019-12-07T19:33:00.950897+0000 allocated 0 extents []) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_link db/LOCK to 2 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0x0 mtime 2019-12-07T19:33:00.950915+0000 allocated 0 extents []) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_dir_link db/MANIFEST-000001 to 3 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x1000: op_file_update file(ino 3 size 0xd mtime 2019-12-07T19:33:00.950942+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x2000~1000 from file(ino 1 size 0x2000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xfe000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0x2000: txn(seq 3 len 0x57 crc 0x27b7df6c) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x0 mtime 2019-12-07T19:33:00.952418+0000 allocated 0 extents []) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x2000: op_dir_link db/000001.dbtmp to 4 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x2000: op_file_update file(ino 4 size 0x10 mtime 2019-12-07T19:33:00.952430+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x3000~1000 from file(ino 1 size 0x3000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xfd000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0x3000: txn(seq 4 len 0x31 crc 0x14e7d54d) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x3000: op_dir_link db/CURRENT to 4 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x3000: op_dir_unlink db/000001.dbtmp 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x4000~1000 from file(ino 1 size 0x4000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xfc000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0x4000: txn(seq 5 len 0x57 crc 0x14583d45) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x0 mtime 2019-12-07T19:33:00.953767+0000 allocated 0 extents []) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x4000: op_dir_link db/000000.dbtmp to 5 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x4000: op_file_update file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x5000~1000 from file(ino 1 size 0x5000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xfb000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0x5000: txn(seq 6 len 0x69 crc 0x2d04f952) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x5000: op_dir_link db/IDENTITY to 5 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x5000: op_dir_unlink db/000000.dbtmp 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x5000: op_file_update file(ino 6 size 0x0 mtime 2019-12-07T19:33:00.956937+0000 allocated 0 extents []) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x5000: op_dir_link db.wal/000003.log to 6 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x6000~1000 from file(ino 1 size 0x6000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xfa000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0x6000: txn(seq 7 len 0x60 crc 0xb01de134) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x0 mtime 2019-12-07T19:33:00.957243+0000 allocated 0 extents []) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x6000: op_dir_link db/OPTIONS-000004.dbtmp to 7 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x6000: op_file_update file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x7000~1000 from file(ino 1 size 0x7000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xf9000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0x7000: txn(seq 8 len 0x64 crc 0xd04f7e35) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x7000: op_dir_link db/OPTIONS-000005 to 7 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x7000: op_dir_unlink db/OPTIONS-000004.dbtmp 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x7000: op_file_update file(ino 6 size 0x10000 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x8000~1000 from file(ino 1 size 0x8000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xf8000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0x8000: txn(seq 9 len 0x23 crc 0x6f84983f) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x8000: op_file_update file(ino 6 size 0x1a1 mtime 2019-12-07T19:33:00.958244+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x9000~1000 from file(ino 1 size 0x9000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xf7000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0x9000: txn(seq 10 len 0x56 crc 0x9f95ef2d) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x0 mtime 2019-12-07T19:33:02.715439+0000 allocated 0 extents []) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x9000: op_dir_link db/000004.sst to 8 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0x9000: op_file_update file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0xa000~1000 from file(ino 1 size 0xa000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xf6000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0xa000: txn(seq 11 len 0x5a crc 0xeeb8e4c4) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x0 mtime 2019-12-07T19:33:02.716009+0000 allocated 0 extents []) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xa000: op_dir_link db/MANIFEST-000005 to 9 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xa000: op_file_update file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0xb000~1000 from file(ino 1 size 0xb000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xf5000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0xb000: txn(seq 12 len 0x57 crc 0x259a0ad3) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x0 mtime 2019-12-07T19:33:02.716280+0000 allocated 0 extents []) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xb000: op_dir_link db/000005.dbtmp to 10 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xb000: op_file_update file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0xc000~1000 from file(ino 1 size 0xc000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xf4000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0xc000: txn(seq 13 len 0xc8 crc 0xad01a53a) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_unlink db/CURRENT 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_file_remove 4 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_link db/CURRENT to 10 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_unlink db/000005.dbtmp 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_file_update file(ino 11 size 0x0 mtime 2019-12-07T19:33:02.716499+0000 allocated 0 extents []) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_link db.wal/000006.log to 11 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_unlink db.wal/000003.log 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_file_remove 6 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_dir_unlink db/MANIFEST-000001 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xc000: op_file_remove 3 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0xd000~1000 from file(ino 1 size 0xd000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xf3000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0xd000: txn(seq 14 len 0x60 crc 0x56fcbb5d) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x0 mtime 2019-12-07T19:33:02.716642+0000 allocated 0 extents []) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xd000: op_dir_link db/OPTIONS-000007.dbtmp to 12 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xd000: op_file_update file(ino 12 size 0x1355 mtime 2019-12-07T19:33:02.716978+0000 allocated 10000 extents [1:0xacd0c0000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0xe000~1000 from file(ino 1 size 0xe000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xf2000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0xe000: txn(seq 15 len 0x64 crc 0xc7ef94bb) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xe000: op_dir_link db/OPTIONS-000008 to 12 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xe000: op_dir_unlink db/OPTIONS-000007.dbtmp 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xe000: op_file_update file(ino 11 size 0x10000 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0xf000~1000 from file(ino 1 size 0xf000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xf1000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0xf000: txn(seq 16 len 0x23 crc 0x81791f6f) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _replay 0xf000: op_file_update file(ino 11 size 0x120 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x10000~1000 from file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0xf0000 len 0x1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 4096 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay 0x10000: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid db82411a-1632-4be0-8c82-a647203179ea 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay log file size was 0x10000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _replay done 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacd0d0000~10000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacd130000~10000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacd120000~10000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacd110000~10000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacd0f0000~10000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacd0e0000~10000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacd0c0000~10000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free 0xacccc0000~400000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 init_rm_free done 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs mount log write pos set to 0x10000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs maybe_verify_layout bluefs layout verified positively 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family L: 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family M: 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) column family P: 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option compaction_readahead_size = 2097152 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option compression = kNoCompression 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option max_background_compactions = 2 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option max_write_buffer_number = 4 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option min_write_buffer_number_to_merge = 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option recycle_log_file_num = 4 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option writable_file_max_buffer_size = 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 0 set rocksdb option write_buffer_size = 268435456 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 rocksdb: load_rocksdb_options using custom Env 0x558e80e6b4e0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 rocksdb: load_rocksdb_options set bloom filter bits per key to 20 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 rocksdb: load_rocksdb_options block size 4096, block_cache size 1.2 GiB, row_cache size 0 B; shards 16, type binned_lru 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80203340 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 16 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80203340 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 102 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 1 rocksdb: do_open column families: [default] 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs readdir db 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: RocksDB version: 6.1.2 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Compile date Dec 5 2019 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: DB SUMMARY 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs readdir db 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: CURRENT file: CURRENT 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: IDENTITY file: IDENTITY 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 102 Bytes 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: SST files in db dir, Total Num: 1, files: 000004.sst 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs stat db.wal/000006.log 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs stat db.wal/000006.log file(ino 11 size 0x120 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Write Ahead Log file in db.wal: 000006.log size: 288 ; 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.error_if_exists: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.create_if_missing: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.paranoid_checks: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.env: 0x558e80e6b4e0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.info_log: 0x558e80e6d360 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_file_opening_threads: 16 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.statistics: (nil) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.use_fsync: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_log_file_size: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.log_file_time_to_roll: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.keep_log_file_num: 1000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.recycle_log_file_num: 4 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_fallocate: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_mmap_reads: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_mmap_writes: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.use_direct_reads: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.create_missing_column_families: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.db_log_dir: 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.wal_dir: db.wal 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.table_cache_numshardbits: 6 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_subcompactions: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_background_flushes: -1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.WAL_ttl_seconds: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.WAL_size_limit_MB: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.is_fd_close_on_exec: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.advise_random_on_open: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.db_write_buffer_size: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.write_buffer_manager: 0x558e81001920 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.use_adaptive_mutex: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.rate_limiter: (nil) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.wal_recovery_mode: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.enable_thread_tracking: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.enable_pipelined_write: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.row_cache: None 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.wal_filter: None 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.allow_ingest_behind: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.preserve_deletes: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.two_write_queues: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.manual_wal_flush: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.atomic_flush: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_background_jobs: 2 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_background_compactions: 2 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.writable_file_max_buffer_size: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.delayed_write_rate : 16777216 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_total_wal_size: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.stats_dump_period_sec: 600 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.stats_persist_period_sec: 600 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_open_files: -1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.bytes_per_sync: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.wal_bytes_per_sync: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_readahead_size: 2097152 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Compression algorithms supported: 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: kZSTD supported: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: kXpressCompression supported: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: kLZ4HCCompression supported: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: kLZ4Compression supported: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: kBZip2Compression supported: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: kZlibCompression supported: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: kSnappyCompression supported: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Fast CRC32 supported: Supported on x86 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs mkdir db.wal 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs mkdir db 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs mkdir db 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs mkdir db.wal 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs mkdir dir db.wal exists 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs dir_exists db.wal = 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs lock_file db/LOCK 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs lock_file locked file(ino 2 size 0x0 mtime 2019-12-07T19:33:00.950897+0000 allocated 0 extents []) with 0x558e801ff7c8 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs stat db/CURRENT 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs stat db/CURRENT file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs stat db/IDENTITY 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs stat db/IDENTITY file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs open_for_read db/IDENTITY (random) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80203340 on file(ino 5 size 0x25 mtime 2019-12-07T19:33:00.953782+0000 allocated 10000 extents [1:0xacd0e0000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs open_for_read db/CURRENT (sequential) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80203340 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x0~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x10 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd130000~10000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x10 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 16 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x10~2000 from file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-000005 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs open_for_read db/MANIFEST-000005 (sequential) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80203340 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs stat db/MANIFEST-000005 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs stat db/MANIFEST-000005 file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_filter: None 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_filter_factory: None 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_factory: SkipListFactory 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.table_factory: BlockBasedTable 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558e801ff770) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x558e80233410 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.write_buffer_size: 268435456 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_write_buffer_number: 4 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compression: NoCompression 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression: Disabled 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.prefix_extractor: nullptr 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.num_levels: 7 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.window_bits: -14 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.level: 32767 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.strategy: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compression_opts.enabled: false 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.target_file_size_base: 67108864 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.target_file_size_multiplier: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.arena_block_size: 33554432 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.disable_auto_compactions: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.table_properties_collectors: 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.inplace_update_support: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.inplace_update_num_locks: 10000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.memtable_huge_page_size: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.bloom_locality: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.max_successive_merges: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.optimize_filters_for_hits: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.paranoid_file_checks: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.force_consistency_checks: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.report_bg_io_stats: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 4 rocksdb: Options.ttl: 0 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203340 0x0~8000 from file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x66 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd120000~10000 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x66 2019-12-07T19:33:36.539+0000 7fe2ed47bbc0 20 bluefs _read got 102 2019-12-07T19:33:36.539+0000 7fe2d5a82700 10 bluefs open_for_read db/000004.sst (random) 2019-12-07T19:33:36.539+0000 7fe2d5a82700 10 bluefs open_for_read h 0x558e80202000 on file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.539+0000 7fe2d5a82700 10 bluefs _read h 0x558e80202000 0x0~567 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) prefetch 2019-12-07T19:33:36.539+0000 7fe2d5a82700 20 bluefs _read fetching 0x0~1000 of 1:0xacd110000~10000 2019-12-07T19:33:36.539+0000 7fe2d5a82700 20 bluefs _read left 0x1000 len 0x567 2019-12-07T19:33:36.539+0000 7fe2d5a82700 20 bluefs _read got 1383 2019-12-07T19:33:36.539+0000 7fe2d5a82700 10 bluefs _read_random h 0x558e80202000 0x532~35 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.540+0000 7fe2d5a82700 20 bluefs _read_random left 0xace 0x532~35 2019-12-07T19:33:36.540+0000 7fe2d5a82700 20 bluefs _read_random got 53 2019-12-07T19:33:36.540+0000 7fe2d5a82700 10 bluefs _read_random h 0x558e80202000 0x4dd~55 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.540+0000 7fe2d5a82700 20 bluefs _read_random left 0xb23 0x4dd~55 2019-12-07T19:33:36.540+0000 7fe2d5a82700 20 bluefs _read_random got 85 2019-12-07T19:33:36.540+0000 7fe2d5a82700 10 bluefs _read_random h 0x558e80202000 0x225~2b8 from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.540+0000 7fe2d5a82700 20 bluefs _read_random left 0xddb 0x225~2b8 2019-12-07T19:33:36.540+0000 7fe2d5a82700 20 bluefs _read_random got 696 2019-12-07T19:33:36.540+0000 7fe2d5a82700 10 bluefs _read_random h 0x558e80202000 0x209~1c from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.540+0000 7fe2d5a82700 20 bluefs _read_random left 0xdf7 0x209~1c 2019-12-07T19:33:36.540+0000 7fe2d5a82700 20 bluefs _read_random got 28 2019-12-07T19:33:36.540+0000 7fe2d5a82700 10 bluefs _read_random h 0x558e80202000 0x1bf~4a from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.540+0000 7fe2d5a82700 20 bluefs _read_random left 0xe41 0x1bf~4a 2019-12-07T19:33:36.540+0000 7fe2d5a82700 20 bluefs _read_random got 74 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 14, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 4 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs mkdir db 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs mkdir dir db exists 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs dir_exists db = 1 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216541982, "job": 1, "event": "recovery_started", "log_files": [6]} 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #6 mode 0 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs open_for_read db.wal/000006.log (sequential) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80203550 on file(ino 11 size 0x120 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203550 0x0~8000 from file(ino 11 size 0x120 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x120 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd0d0000~10000 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read left 0x1000 len 0x120 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read got 288 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs open_for_write db/000007.sst 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs open_for_write mapping db/000007.sst vsel_hint 0x1 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs open_for_write h 0x558e8107a140 on file(ino 13 size 0x0 mtime 2019-12-07T19:33:36.542019+0000 allocated 0 extents []) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e8107a140 ignoring, length 1238 < min_flush_size 524288 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _fsync 0x558e8107a140 file(ino 13 size 0x0 mtime 2019-12-07T19:33:36.542019+0000 allocated 0 extents []) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e8107a140 0x0~4d6 to file(ino 13 size 0x0 mtime 2019-12-07T19:33:36.542019+0000 allocated 0 extents []) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush_range 0x558e8107a140 pos 0x0 0x0~4d6 to file(ino 13 size 0x0 mtime 2019-12-07T19:33:36.542019+0000 allocated 0 extents []) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _allocate len 0x4d6 from 1 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd100000~10000/10000,10000,0 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range dirty_seq = 17 (was clean) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range file now file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.542106+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range in 1:0xacd100000~10000 x_off 0x0 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range caching tail of 0x4d6 and padding block with 0xb2a 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range h 0x558e8107a140 pos now 0x4d6 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e8107a140 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e8107a140 done in 0.000000 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs flush_bdev 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _fsync file metadata was dirty (17) on file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.542106+0000 allocated 10000 extents [1:0xacd100000~10000]), flushing log 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.542106+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush_and_sync_log txn(seq 17 len 0x56 crc 0xb2ca9974) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e803a5a40 0x10000~1000 to file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush_range 0x558e803a5a40 pos 0x10000 0x10000~1000 to file(ino 1 size 0x10000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range file now file(ino 1 size 0x11000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x10000 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x11000 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000019 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs flush_bdev 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log log_seq_stable 17 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.542106+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e8107a140 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e8107a140 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _close_writer 0x558e8107a140 type 2 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs open_for_read db/000007.sst (random) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e80203550 on file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.542106+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e80203550 0x194~342 from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.542106+0000 allocated 10000 extents [1:0xacd100000~10000]) prefetch 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~1000 of 1:0xacd100000~10000 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read left 0xe6c len 0x342 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read got 834 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _read_random h 0x558e80203550 0x4a1~35 from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.542106+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read_random left 0xb5f 0x4a1~35 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read_random got 53 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _read_random h 0x558e80203550 0x44c~55 from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.542106+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read_random left 0xbb4 0x44c~55 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read_random got 85 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _read_random h 0x558e80203550 0x193~2b9 from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.542106+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read_random left 0xe6d 0x193~2b9 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read_random got 697 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _read_random h 0x558e80203550 0x177~1c from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.542106+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read_random left 0xe89 0x177~1c 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read_random got 28 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _read_random h 0x558e80203550 0x12d~4a from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.542106+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read_random left 0xed3 0x12d~4a 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _read_random got 74 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216542372, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 1238, "table_properties": {"data_size": 301, "index_size": 28, "filter_size": 69, "raw_key_size": 144, "raw_average_key_size": 24, "raw_value_size": 128, "raw_average_value_size": 21, "num_data_blocks": 1, "num_entries": 6, "filter_policy_name": "rocksdb.BuiltinBloomFilter"}} 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 4 rocksdb: [db/version_set.cc:3036] Creating manifest 8 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs open_for_write db/MANIFEST-000008 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs open_for_write mapping db/MANIFEST-000008 vsel_hint 0x1 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs open_for_write h 0x558e8107a140 on file(ino 14 size 0x0 mtime 2019-12-07T19:33:36.542403+0000 allocated 0 extents []) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e8107a140 ignoring, length 35 < min_flush_size 524288 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e8107a140 ignoring, length 87 < min_flush_size 524288 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e8107a140 ignoring, length 143 < min_flush_size 524288 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e8107a140 ignoring, length 143 < min_flush_size 524288 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _fsync 0x558e8107a140 file(ino 14 size 0x0 mtime 2019-12-07T19:33:36.542403+0000 allocated 0 extents []) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e8107a140 0x0~8f to file(ino 14 size 0x0 mtime 2019-12-07T19:33:36.542403+0000 allocated 0 extents []) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush_range 0x558e8107a140 pos 0x0 0x0~8f to file(ino 14 size 0x0 mtime 2019-12-07T19:33:36.542403+0000 allocated 0 extents []) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _allocate len 0x8f from 1 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd140000~10000/10000,10000,0 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range dirty_seq = 18 (was clean) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range file now file(ino 14 size 0x8f mtime 2019-12-07T19:33:36.542427+0000 allocated 10000 extents [1:0xacd140000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range in 1:0xacd140000~10000 x_off 0x0 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range caching tail of 0x8f and padding block with 0xf71 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range h 0x558e8107a140 pos now 0x8f 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e8107a140 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e8107a140 done in 0.000000 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs flush_bdev 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _fsync file metadata was dirty (18) on file(ino 14 size 0x8f mtime 2019-12-07T19:33:36.542427+0000 allocated 10000 extents [1:0xacd140000~10000]), flushing log 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 14 size 0x8f mtime 2019-12-07T19:33:36.542427+0000 allocated 10000 extents [1:0xacd140000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush_and_sync_log txn(seq 18 len 0x5b crc 0xd20f287e) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e803a5a40 0x11000~1000 to file(ino 1 size 0x11000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush_range 0x558e803a5a40 pos 0x11000 0x11000~1000 to file(ino 1 size 0x11000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range file now file(ino 1 size 0x12000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x11000 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x12000 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000015 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs flush_bdev 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log log_seq_stable 18 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 14 size 0x8f mtime 2019-12-07T19:33:36.542427+0000 allocated 10000 extents [1:0xacd140000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs open_for_write db/000008.dbtmp 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs open_for_write mapping db/000008.dbtmp vsel_hint 0x1 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs open_for_write h 0x558e8107a280 on file(ino 15 size 0x0 mtime 2019-12-07T19:33:36.542649+0000 allocated 0 extents []) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _fsync 0x558e8107a280 file(ino 15 size 0x0 mtime 2019-12-07T19:33:36.542649+0000 allocated 0 extents []) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e8107a280 0x0~10 to file(ino 15 size 0x0 mtime 2019-12-07T19:33:36.542649+0000 allocated 0 extents []) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _flush_range 0x558e8107a280 pos 0x0 0x0~10 to file(ino 15 size 0x0 mtime 2019-12-07T19:33:36.542649+0000 allocated 0 extents []) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 bluefs _allocate len 0x10 from 1 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd150000~10000/10000,10000,0 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range dirty_seq = 19 (was clean) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range file now file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.542661+0000 allocated 10000 extents [1:0xacd150000~10000]) 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range in 1:0xacd150000~10000 x_off 0x0 2019-12-07T19:33:36.540+0000 7fe2ed47bbc0 20 bluefs _flush_range caching tail of 0x10 and padding block with 0xff0 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 20 bluefs _flush_range h 0x558e8107a280 pos now 0x10 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 20 bluefs flush_bdev 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 20 bluefs _fsync file metadata was dirty (19) on file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.542661+0000 allocated 10000 extents [1:0xacd150000~10000]), flushing log 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.542661+0000 allocated 10000 extents [1:0xacd150000~10000]) 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 10 bluefs _flush_and_sync_log txn(seq 19 len 0x57 crc 0x51986094) 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e803a5a40 0x12000~1000 to file(ino 1 size 0x12000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 10 bluefs _flush_range 0x558e803a5a40 pos 0x12000 0x12000~1000 to file(ino 1 size 0x12000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 20 bluefs _flush_range file now file(ino 1 size 0x13000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x12000 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x13000 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.541+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.001027 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs flush_bdev 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log log_seq_stable 19 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.542661+0000 allocated 10000 extents [1:0xacd150000~10000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _close_writer 0x558e8107a280 type 0 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs rename db/000008.dbtmp -> db/CURRENT 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs rename dir db (0x558e80f091a0) file CURRENT already exists, unlinking 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _drop_link had refs 1 on file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _drop_link destroying file(ino 10 size 0x10 mtime 2019-12-07T19:33:02.716295+0000 allocated 10000 extents [1:0xacd130000~10000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs rename db/CURRENT file(ino 15 size 0x10 mtime 2019-12-07T19:33:36.542661+0000 allocated 10000 extents [1:0xacd150000~10000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1575747216543925, "job": 1, "event": "recovery_finished"} 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs open_for_write db.wal/000009.log 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs open_for_write mapping db.wal/000009.log vsel_hint 0 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs open_for_write h 0x558e8107a280 on file(ino 16 size 0x0 mtime 2019-12-07T19:33:36.543936+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs readdir db 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs readdir db.wal 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs unlink db.wal/000006.log 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _drop_link had refs 1 on file(ino 11 size 0x120 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _drop_link destroying file(ino 11 size 0x120 mtime 2019-12-07T19:33:02.718448+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs unlink db/MANIFEST-000005 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _drop_link had refs 1 on file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _drop_link destroying file(ino 9 size 0x66 mtime 2019-12-07T19:33:02.716036+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs sync_metadata 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs flush_bdev 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _flush_and_sync_log txn(seq 20 len 0xc8 crc 0x45f3d5f4) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e803a5a40 0x13000~1000 to file(ino 1 size 0x13000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _flush_range 0x558e803a5a40 pos 0x13000 0x13000~1000 to file(ino 1 size 0x13000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_range file now file(ino 1 size 0x14000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x13000 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x14000 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000017 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs flush_bdev 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log log_seq_stable 20 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 release 0xacd0d0000~10000 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 release 0xacd120000~20000 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 release done 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs sync_metadata done in 0.000108 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _should_compact_log current 0x14000 expected 3000 ratio 6.66667 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs open_for_write db/OPTIONS-000010.dbtmp 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs open_for_write mapping db/OPTIONS-000010.dbtmp vsel_hint 0x1 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs open_for_write h 0x558e8107a3c0 on file(ino 17 size 0x0 mtime 2019-12-07T19:33:36.544124+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e8107a3c0 ignoring, length 4949 < min_flush_size 524288 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _fsync 0x558e8107a3c0 file(ino 17 size 0x0 mtime 2019-12-07T19:33:36.544124+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e8107a3c0 0x0~1355 to file(ino 17 size 0x0 mtime 2019-12-07T19:33:36.544124+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _flush_range 0x558e8107a3c0 pos 0x0 0x0~1355 to file(ino 17 size 0x0 mtime 2019-12-07T19:33:36.544124+0000 allocated 0 extents []) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _allocate len 0x1355 from 1 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd0d0000~10000/10000,10000,0 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_range dirty_seq = 21 (was clean) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_range file now file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.544455+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_range in 1:0xacd0d0000~10000 x_off 0x0 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_range caching tail of 0x355 and padding block with 0xcab 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_range h 0x558e8107a3c0 pos now 0x1355 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e8107a3c0 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e8107a3c0 done in 0.000000 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs flush_bdev 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _fsync file metadata was dirty (21) on file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.544455+0000 allocated 10000 extents [1:0xacd0d0000~10000]), flushing log 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log op_file_update file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.544455+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _flush_and_sync_log txn(seq 21 len 0x60 crc 0x71d3f507) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e803a5a40 0x14000~1000 to file(ino 1 size 0x14000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _flush_range 0x558e803a5a40 pos 0x14000 0x14000~1000 to file(ino 1 size 0x14000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_range file now file(ino 1 size 0x15000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x14000 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x15000 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000018 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs flush_bdev 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log log_seq_stable 21 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _flush_and_sync_log cleaned file file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.544455+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e8107a3c0 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _flush 0x558e8107a3c0 ignoring, length 0 < min_flush_size 524288 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _close_writer 0x558e8107a3c0 type 0 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs open_for_read db/OPTIONS-000010.dbtmp (sequential) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs open_for_read h 0x558e802036b0 on file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.544455+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 10 bluefs _read h 0x558e802036b0 0x0~2000 from file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.544455+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _read reaching (or past) eof, len clipped to 0x1355 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _read fetching 0x0~2000 of 1:0xacd0d0000~10000 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _read left 0x2000 len 0x1355 2019-12-07T19:33:36.542+0000 7fe2ed47bbc0 20 bluefs _read got 4949 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluefs rename db/OPTIONS-000010.dbtmp -> db/OPTIONS-000011 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluefs rename db/OPTIONS-000011 file(ino 17 size 0x1355 mtime 2019-12-07T19:33:36.544455+0000 allocated 10000 extents [1:0xacd0d0000~10000]) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluefs readdir db 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluefs unlink db/OPTIONS-000005 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluefs _drop_link had refs 1 on file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluefs _drop_link destroying file(ino 7 size 0x1354 mtime 2019-12-07T19:33:00.957606+0000 allocated 10000 extents [1:0xacd0f0000~10000]) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluefs readdir db 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluefs stat db/000004.sst 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluefs stat db/000004.sst file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluefs stat db/000007.sst 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluefs stat db/000007.sst file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.542106+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 4 rocksdb: DB pointer 0x558e80272400 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _upgrade_super from 3, latest 3 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) _upgrade_super done 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _open_collections 2019-12-07T19:33:36.543+0000 7fe2d6283700 4 rocksdb: [db/db_impl.cc:777] ------- DUMPING STATS ------- 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluefs _read_random h 0x558e80203550 0x0~12d from file(ino 13 size 0x4d6 mtime 2019-12-07T19:33:36.542106+0000 allocated 10000 extents [1:0xacd100000~10000]) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluefs _read_random left 0x1000 0x0~12d 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluefs _read_random got 301 2019-12-07T19:33:36.543+0000 7fe2d6283700 4 rocksdb: [db/db_impl.cc:778] ** DB Stats ** Uptime(secs): 0.0 total, 0.0 interval Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s Interval stall: 00:00:0.000 H:M:S, 0.0 percent ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 2/0 2.56 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 Sum 2/0 2.56 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.1 0.00 0.00 1 0.000 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.35 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.35 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 2/0 2.56 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 Sum 2/0 2.56 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.1 0.00 0.00 1 0.000 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluefs _read_random h 0x558e80202000 0x0~1bf from file(ino 8 size 0x567 mtime 2019-12-07T19:33:02.715575+0000 allocated 10000 extents [1:0xacd110000~10000]) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluefs _read_random left 0x1000 0x0~1bf 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluefs _read_random got 447 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _open_collections opened meta 0x558e80230960 cnode(bits 0) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach meta fresh osr 0x558e8107a500 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x233/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_start 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread start 2019-12-07T19:33:36.543+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_replay start 2019-12-07T19:33:36.543+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread start 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_replay draining osr 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_register_zombie 0x558e8107a500 meta 2019-12-07T19:33:36.543+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_drain_all 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _osr_drain_all osr_set 0x558e8107a500 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:36.543+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _osr_drain_all drain 0x558e8107a500 2019-12-07T19:33:36.543+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:36.543+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:36.543+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_drain_all reaping empty zombie osr 0x558e8107a500 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_drain_all done 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_replay completed 0 events 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) is_journal_rotational 0 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 2 osd.0 0 journal looks like ssd 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 2 osd.0 0 boot 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 osd.0 0 configured osd_max_object_name[space]_len looks ok 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:7b3f43c4:::osd_superblock:0# 0x0~0 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:7b3f43c4:::osd_superblock:0# key 0x7f7fffffffffffffff7b3f43c4216f73'd_superblock!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r 0 v.len 46 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore.blob(0x558e810784d0) get_ref 0x0~233 Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore.blob(0x558e810784d0) get_ref init 0x1000, 1000 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~233 size 0x233 (563) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x558e81078620 sbid 0x0)) need 0x0~233 cache has 0x[] 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x558e81078620 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~233]} 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x558e81078620 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~233]} 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 _add level 1 near 0 on buffer(0x558e80229ef0 space 0x558e81078638 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:7b3f43c4:::osd_superblock:0# 0x0~233 = 563 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 osd.0 0 read_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e0 [0,0] lci=[0,0]) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 osd.0 0 get_map 0 - return initial 0x558e8020f200 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) exists meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c0371625:::snapmapper:0# key 0x7f7fffffffffffffffc03716'%!snapmapper!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 osd.0 0 init creating/touching snapmapper object 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e8106c900 seq 1 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c0371625:::snapmapper:0# key 0x7f7fffffffffffffffc03716'%!snapmapper!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1026 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c900 cost 4205 (1 ios * 4000 + 205 bytes) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c900 onodes 0x558e8125c280 shared_blobs 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore.extentmap(0x558e8125c388) update #-1:c0371625:::snapmapper:0# 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore.extentmap(0x558e8125c388) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c0371625:::snapmapper:0# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c900 allocated 0x[] released 0x[] 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 prepare 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c900 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 io_done 2019-12-07T19:33:36.543+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) exists meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:36.543+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c1a3fc6e:::purged_snaps:0# key 0x7f7fffffffffffffffc1a3fc'n!purged_snaps!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 osd.0 0 init creating/touching purged_snaps object 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e8106c000 seq 2 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread new_nid_max 2050 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread new_blobid_max 20480 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c1a3fc6e:::purged_snaps:0# key 0x7f7fffffffffffffffc1a3fc'n!purged_snaps!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1027 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c000 cost 4207 (1 ios * 4000 + 207 bytes) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c000 onodes 0x558e8125c500 shared_blobs 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore.extentmap(0x558e8125c608) update #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore.extentmap(0x558e8125c608) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 143 < min_flush_size 524288 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c1a3fc6e:::purged_snaps:0# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c000 allocated 0x[] released 0x[] 2019-12-07T19:33:36.543+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:33:36.543+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 0 Request: 0 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 prepare 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c000 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 io_done 2019-12-07T19:33:36.543+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 1 Request: 1760 2019-12-07T19:33:36.543+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 open_all_classes 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:36.543+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:36.543+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:36.543+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:36.543+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:36.543+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:36.543+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:36.543+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:36.543+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:36.543+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959999), bluefs_ratio 0.0399535 2019-12-07T19:33:36.543+0000 7fe2dc28f700 10 rocksdb: commit_cache_size old: 1288490188 new: 1073741824 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 166 < min_flush_size 524288 2019-12-07T19:33:36.543+0000 7fe2dc28f700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0263526 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 166 < min_flush_size 524288 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x0 mtime 2019-12-07T19:33:36.543936+0000 allocated 0 extents []) 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x0~a6 to file(ino 16 size 0x0 mtime 2019-12-07T19:33:36.543936+0000 allocated 0 extents []) 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x0 0x0~a6 to file(ino 16 size 0x0 mtime 2019-12-07T19:33:36.543936+0000 allocated 0 extents []) 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluefs _allocate len 0xa6 from 0 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluefs _allocate len 0xa6 from 1 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,0 2019-12-07T19:33:36.543+0000 7fe2dc28f700 5 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3417 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd120000~10000/10000,10000,0 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x10000 to include allocated 2019-12-07T19:33:36.543+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 22 (was clean) 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 open_all_classes found journal 2019-12-07T19:33:36.543+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.543+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x0 2019-12-07T19:33:36.543+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xa6 and padding block with 0xf5a 2019-12-07T19:33:36.543+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x10000 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 _get_class adding new class name journal 0x558e8023b588 2019-12-07T19:33:36.543+0000 7fe2ed47bbc0 10 _load_class journal from /usr/lib64/rados-classes/libcls_journal.so 2019-12-07T19:33:36.543+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xa6 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:36.543+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:36.543+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (22) on file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]), flushing log 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 22 len 0x86 crc 0x50c44971) 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x15000~1000 to file(ino 1 size 0x15000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x15000 0x15000~1000 to file(ino 1 size 0x15000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x16000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x15000 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x16000 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000017 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 22 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 release 0xacd0f0000~10000 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 release done 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs sync_metadata - no pending log events 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs _should_compact_log current 0x16000 expected 3000 ratio 7.33333 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread nid_max now 2050 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread blobid_max now 20480 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000406475s (0.000000092s flush + 0.000406383s kv commit) 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8106c900> 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:36.544+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 kv_submitted 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c900 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 102 < min_flush_size 524288 2019-12-07T19:33:36.544+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 finishing 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c900 onodes 0x558e8125c280 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c900 done 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c000 kv_submitted 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:33:36.544+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c900 [] 2019-12-07T19:33:36.544+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 125 < min_flush_size 524288 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 125 < min_flush_size 524288 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xa6~7d to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xa6 0xa6~7d to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xa6 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xa6 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x123 and padding block with 0xedd 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_class journal status 3 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.create flags 3 0x7fe2db877130 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.get_order flags 1 0x7fe2db876bf0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.get_splay_width flags 1 0x7fe2db876b90 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.get_pool_id flags 1 0x7fe2db876440 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.get_minimum_set flags 1 0x7fe2db876e80 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.set_minimum_set flags 3 0x7fe2db877010 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.get_active_set flags 1 0x7fe2db876e10 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.set_active_set flags 3 0x7fe2db876ef0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.get_client flags 1 0x7fe2db877d90 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.client_register flags 3 0x7fe2db879890 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.client_update_data flags 3 0x7fe2db8780e0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.client_update_state flags 3 0x7fe2db878660 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.client_unregister flags 3 0x7fe2db87c360 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x123 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.client_commit flags 3 0x7fe2db878a70 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.client_list flags 1 0x7fe2db879590 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.get_next_tag_tid flags 1 0x7fe2db876da0 2019-12-07T19:33:36.544+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.get_tag flags 1 0x7fe2db87cae0 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.tag_create flags 3 0x7fe2db87cd80 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.tag_list flags 1 0x7fe2db87a570 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.guard_append flags 3 0x7fe2db877490 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method journal.append flags 3 0x7fe2db8775d0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 _load_class journal success 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 open_all_classes found version 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000100297s (0.000000084s flush + 0.000100213s kv commit) 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:36.544+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8106c000> 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:36.544+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 kv_submitted 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c000 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 _get_class adding new class name version 0x558e8023b108 2019-12-07T19:33:36.544+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 finishing 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 _load_class version from /usr/lib64/rados-classes/libcls_version.so 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c000 onodes 0x558e8125c500 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c000 done 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107a500 q now empty 2019-12-07T19:33:36.544+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c000 [] 2019-12-07T19:33:36.544+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:36.544+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish empty zombie osr 0x558e8107a500 already reaped 2019-12-07T19:33:36.544+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_class version status 3 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method version.set flags 3 0x7fe2db65bd30 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method version.inc flags 3 0x7fe2db65c740 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method version.inc_conds flags 3 0x7fe2db65c740 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method version.read flags 1 0x7fe2db65bbb0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method version.check_conds flags 1 0x7fe2db65c590 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 _load_class version success 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 open_all_classes found log 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 _get_class adding new class name log 0x558e8023b228 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 _load_class log from /usr/lib64/rados-classes/libcls_log.so 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_class log status 3 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method log.add flags 3 0x7fe2db444c40 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method log.list flags 1 0x7fe2db4434a0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method log.trim flags 3 0x7fe2db4429e0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method log.info flags 1 0x7fe2db4424e0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 _load_class log success 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 open_all_classes found rbd 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 _get_class adding new class name rbd 0x558e8023b348 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 _load_class rbd from /usr/lib64/rados-classes/libcls_rbd.so 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_class rbd status 3 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.create flags 3 0x7fe2db1ebc90 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_features flags 1 0x7fe2db1e1fc0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.set_features flags 3 0x7fe2db1e21a0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_size flags 1 0x7fe2db1f4150 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.set_size flags 3 0x7fe2db1ea200 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_snapcontext flags 1 0x7fe2db1eb300 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_object_prefix flags 1 0x7fe2db1e33b0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_data_pool flags 1 0x7fe2db1e3520 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_snapshot_name flags 1 0x7fe2db1f5a20 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_snapshot_timestamp flags 1 0x7fe2db1f5bf0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.snapshot_get flags 1 0x7fe2db1f6ae0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.snapshot_add flags 3 0x7fe2db1f6ec0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.snapshot_remove flags 3 0x7fe2db1f6430 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.snapshot_rename flags 3 0x7fe2db1f5da0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.snapshot_trash_add flags 3 0x7fe2db1fb0d0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_all_features flags 1 0x7fe2db1d4f70 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_parent flags 1 0x7fe2db1f5150 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.set_parent flags 3 0x7fe2db1e9dc0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.remove_parent flags 3 0x7fe2db1f4110 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.parent_get flags 1 0x7fe2db1eadc0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.parent_overlap_get flags 1 0x7fe2db1f5620 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.parent_attach flags 3 0x7fe2db1e9ff0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.parent_detach flags 3 0x7fe2db1f4130 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.set_protection_status flags 3 0x7fe2db1f4700 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_protection_status flags 1 0x7fe2db1f44c0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_stripe_unit_count flags 1 0x7fe2db1e2540 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.set_stripe_unit_count flags 3 0x7fe2db1e27c0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_create_timestamp flags 1 0x7fe2db1e2c40 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_access_timestamp flags 1 0x7fe2db1e2e50 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_modify_timestamp flags 1 0x7fe2db1e3060 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_flags flags 1 0x7fe2db1f49a0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.set_flags flags 3 0x7fe2db1f4c60 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.op_features_get flags 1 0x7fe2db1e3270 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.op_features_set flags 3 0x7fe2db1e0850 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.metadata_list flags 1 0x7fe2db1eda00 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.metadata_set flags 3 0x7fe2db1fb3d0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.metadata_remove flags 3 0x7fe2db1da4e0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.metadata_get flags 1 0x7fe2db1da640 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.snapshot_get_limit flags 1 0x7fe2db1e3c20 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.snapshot_set_limit flags 3 0x7fe2db1e4d20 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.child_attach flags 3 0x7fe2db1fa310 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.child_detach flags 3 0x7fe2db1f9710 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.children_list flags 1 0x7fe2db1fa030 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.migration_set flags 3 0x7fe2db1e1070 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.migration_set_state flags 3 0x7fe2db1e1730 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.migration_get flags 1 0x7fe2db1e1920 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.migration_remove flags 3 0x7fe2db1e1fa0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.set_modify_timestamp flags 3 0x7fe2db1e38f0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.set_access_timestamp flags 3 0x7fe2db1e3800 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.add_child flags 3 0x7fe2db1fcd90 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.remove_child flags 3 0x7fe2db1fd170 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_children flags 1 0x7fe2db1fd5d0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.get_id flags 1 0x7fe2db1d6d20 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.set_id flags 3 0x7fe2db1d6f30 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.dir_get_id flags 1 0x7fe2db1d98a0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.dir_get_name flags 1 0x7fe2db1d9eb0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.dir_list flags 1 0x7fe2db1e6970 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.dir_add_image flags 3 0x7fe2db1ecec0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.dir_remove_image flags 3 0x7fe2db1d9dc0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.dir_rename_image flags 3 0x7fe2db1ecd80 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.dir_state_assert flags 1 0x7fe2db1e39e0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.dir_state_set flags 3 0x7fe2db1e5130 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.object_map_load flags 1 0x7fe2db1ee690 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.object_map_save flags 3 0x7fe2db1eeba0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.object_map_resize flags 3 0x7fe2db1efe60 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.object_map_update flags 3 0x7fe2db1ef410 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.object_map_snap_add flags 3 0x7fe2db1ee760 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.object_map_snap_remove flags 3 0x7fe2db1eed40 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.snap_list flags 1 0x7fe2db1e3d60 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.snap_add flags 3 0x7fe2db1e4000 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.snap_remove flags 3 0x7fe2db1d7110 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.snap_rename flags 3 0x7fe2db1d7650 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_uuid_get flags 1 0x7fe2db1e89d0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_uuid_set flags 3 0x7fe2db1eba20 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_mode_get flags 1 0x7fe2db1d5d60 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_mode_set flags 3 0x7fe2db1e9070 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_peer_ping flags 3 0x7fe2db1fafc0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_peer_list flags 1 0x7fe2db1e5ab0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_peer_add flags 3 0x7fe2db1e8dd0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_peer_remove flags 3 0x7fe2db1dae10 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_peer_set_client flags 3 0x7fe2db1dd580 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_peer_set_cluster flags 3 0x7fe2db1e5d50 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_peer_set_direction flags 3 0x7fe2db1dac10 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_list flags 1 0x7fe2db1df230 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_get_image_id flags 1 0x7fe2db1db880 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_get flags 1 0x7fe2db1db0e0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_set flags 3 0x7fe2db1db780 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_remove flags 3 0x7fe2db1e8550 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_status_set flags 11 0x7fe2db1e96c0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_status_remove flags 3 0x7fe2db1e8610 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_status_get flags 1 0x7fe2db1e6360 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_status_list flags 1 0x7fe2db1f1210 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_status_get_summary flags 1 0x7fe2db1f1970 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_status_remove_down flags 3 0x7fe2db1df210 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_instance_get flags 1 0x7fe2db1dd3c0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_instance_list flags 1 0x7fe2db1f23e0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_instances_list flags 1 0x7fe2db1e67b0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_instances_add flags 11 0x7fe2db1dbc20 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_instances_remove flags 3 0x7fe2db1dbdc0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_map_list flags 1 0x7fe2db1de300 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_map_update flags 2 0x7fe2db1dbed0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.mirror_image_map_remove flags 2 0x7fe2db1dc1b0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.group_dir_list flags 1 0x7fe2db1e71d0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.group_dir_add flags 3 0x7fe2db1ed420 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.group_dir_remove flags 3 0x7fe2db1da3f0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.group_dir_rename flags 3 0x7fe2db1ed560 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.group_image_remove flags 3 0x7fe2db1d5330 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.group_image_list flags 1 0x7fe2db1dfb80 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.group_image_set flags 3 0x7fe2db1d5dc0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.image_group_add flags 3 0x7fe2db1e4550 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.image_group_remove flags 3 0x7fe2db1e48b0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.image_group_get flags 1 0x7fe2db1e4b60 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.group_snap_set flags 3 0x7fe2db1f8cf0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.group_snap_remove flags 3 0x7fe2db1f7f50 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.group_snap_get_by_id flags 1 0x7fe2db1f8080 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.group_snap_list flags 1 0x7fe2db1f89e0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.trash_add flags 3 0x7fe2db1ed6a0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.trash_remove flags 3 0x7fe2db1dc340 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.trash_list flags 1 0x7fe2db1f2870 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.trash_get flags 1 0x7fe2db1dc5b0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.trash_state_set flags 3 0x7fe2db1dc7d0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.namespace_add flags 3 0x7fe2db1dcbe0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.namespace_remove flags 3 0x7fe2db1dcdc0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.namespace_list flags 1 0x7fe2db1de4a0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.copyup flags 3 0x7fe2db1d5130 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.sparse_copyup flags 3 0x7fe2db1fc0b0 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.assert_snapc_seq flags 3 0x7fe2db1d5010 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 register_cxx_method rbd.sparsify flags 3 0x7fe2db1d6410 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 _load_class rbd success 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 open_all_classes found sdk 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 0 _get_class not permitted to load sdk 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 open_all_classes found rgw 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 _get_class adding new class name rgw 0x558e8023b468 2019-12-07T19:33:36.544+0000 7fe2ed47bbc0 10 _load_class rgw from /usr/lib64/rados-classes/libcls_rgw.so 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_class rgw status 3 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bucket_init_index flags 3 0x7fe2daed8600 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bucket_set_tag_timeout flags 3 0x7fe2daee0340 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bucket_list flags 1 0x7fe2daee6af0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bucket_check_index flags 1 0x7fe2daee85f0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bucket_rebuild_index flags 3 0x7fe2daee8470 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bucket_update_stats flags 3 0x7fe2daee12c0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bucket_prepare_op flags 3 0x7fe2daee9000 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bucket_complete_op flags 3 0x7fe2daee9520 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bucket_link_olh flags 3 0x7fe2daeeb080 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bucket_unlink_instance flags 3 0x7fe2daeea570 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bucket_read_olh_log flags 1 0x7fe2daee30f0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bucket_trim_olh_log flags 3 0x7fe2daee2ae0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bucket_clear_olh flags 3 0x7fe2daee2600 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.obj_remove flags 3 0x7fe2daee5c90 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.obj_store_pg_ver flags 2 0x7fe2daedb600 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.obj_check_attrs_prefix flags 1 0x7fe2daedb3c0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.obj_check_mtime flags 1 0x7fe2daedb140 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bi_get flags 1 0x7fe2daedd0b0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bi_put flags 3 0x7fe2daedafc0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bi_list flags 1 0x7fe2daee3a10 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bi_log_list flags 1 0x7fe2daeebd60 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bi_log_trim flags 3 0x7fe2daedd600 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.dir_suggest_changes flags 3 0x7fe2daee1570 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bi_log_resync flags 3 0x7fe2daee0ea0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.bi_log_stop flags 3 0x7fe2daee0b60 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.user_usage_log_add flags 3 0x7fe2daedf330 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.user_usage_log_read flags 1 0x7fe2daeded90 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.user_usage_log_trim flags 3 0x7fe2daedf1b0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.usage_log_clear flags 2 0x7fe2daed74d0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.gc_set_entry flags 3 0x7fe2daedcc90 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.gc_defer_entry flags 3 0x7fe2daedcb70 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.gc_list flags 1 0x7fe2daedba80 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.gc_remove flags 3 0x7fe2daee8d20 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.lc_get_entry flags 1 0x7fe2daedad20 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.lc_set_entry flags 3 0x7fe2daedb870 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.lc_rm_entry flags 3 0x7fe2daedb7b0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.lc_get_next_entry flags 1 0x7fe2daedaa30 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.lc_put_head flags 3 0x7fe2daeda860 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.lc_get_head flags 1 0x7fe2daeda610 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.lc_list_entries flags 1 0x7fe2daeda070 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.reshard_add flags 3 0x7fe2daed9ea0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.reshard_list flags 1 0x7fe2daed98e0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.reshard_get flags 1 0x7fe2daed96b0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.reshard_remove flags 3 0x7fe2daed94b0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.set_bucket_resharding flags 3 0x7fe2daee09a0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.clear_bucket_resharding flags 3 0x7fe2daee0840 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.guard_bucket_resharding flags 1 0x7fe2daee06f0 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 register_cxx_method rgw.get_bucket_resharding flags 1 0x7fe2daee0490 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 _load_class rgw success 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 open_all_classes found kvs 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 0 _get_class not permitted to load kvs 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 open_all_classes found refcount 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 _get_class adding new class name refcount 0x558e8023ada8 2019-12-07T19:33:36.546+0000 7fe2ed47bbc0 10 _load_class refcount from /usr/lib64/rados-classes/libcls_refcount.so 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_class refcount status 3 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method refcount.get flags 3 0x7fe2dab882e0 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method refcount.put flags 3 0x7fe2dab88a10 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method refcount.set flags 3 0x7fe2dab87d90 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method refcount.read flags 1 0x7fe2dab89280 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _load_class refcount success 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 open_all_classes found user 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _get_class adding new class name user 0x558e8023aec8 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _load_class user from /usr/lib64/rados-classes/libcls_user.so 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_class user status 3 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method user.set_buckets_info flags 3 0x7fe2da8c7800 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method user.complete_stats_sync flags 3 0x7fe2da8c3d50 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method user.remove_bucket flags 3 0x7fe2da8c3070 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method user.list_buckets flags 1 0x7fe2da8c5930 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method user.get_header flags 1 0x7fe2da8c2b60 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method user.reset_user_stats flags 3 0x7fe2da8c4360 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _load_class user success 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 open_all_classes found hello 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _get_class adding new class name hello 0x558e8023bb28 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _load_class hello from /usr/lib64/rados-classes/libcls_hello.so 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 0 /home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/15.0.0-8038-g503c226/rpm/el7/BUILD/ceph-15.0.0-8038-g503c226/src/cls/hello/cls_hello.cc:313: loading cls_hello 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_class hello status 3 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method hello.say_hello flags 1 0x7fe2da6aa4d0 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method hello.record_hello flags 10 0x7fe2da6ab360 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method hello.write_return_data flags 2 0x7fe2da6aa670 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method hello.write_too_much_return_data flags 2 0x7fe2da6aa550 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method hello.replay flags 1 0x7fe2da6aa4a0 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method hello.turn_it_to_11 flags 11 0x7fe2da6aa9a0 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method hello.bad_reader flags 2 0x7fe2da6aa490 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method hello.bad_writer flags 1 0x7fe2da6aa480 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _load_class hello success 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 open_all_classes found otp 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _get_class adding new class name otp 0x558e8023bd68 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _load_class otp from /usr/lib64/rados-classes/libcls_otp.so 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_class otp status 3 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method otp.otp_set flags 3 0x7fe2da48d4c0 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method otp.otp_get flags 1 0x7fe2da48edd0 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method otp.otp_check flags 3 0x7fe2da48bd50 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method otp.otp_get_result flags 1 0x7fe2da48c6d0 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method otp.otp_remove flags 3 0x7fe2da490380 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method otp.get_current_time flags 1 0x7fe2da48b740 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _load_class otp success 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 open_all_classes found cas 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _get_class adding new class name cas 0x558e8023be88 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _load_class cas from /usr/lib64/rados-classes/libcls_cas.so 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_class cas status 3 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method cas.chunk_get flags 3 0x7fe2da05c570 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method cas.chunk_put flags 3 0x7fe2da05d230 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method cas.chunk_set flags 3 0x7fe2da05db60 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method cas.chunk_read flags 1 0x7fe2da05cb30 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method cas.cas_write_or_get flags 3 0x7fe2da05ccc0 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 register_cxx_method cas.has_chunk flags 1 0x7fe2da05baa0 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _load_class cas success 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 open_all_classes found rgw_gc 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _get_class adding new class name rgw_gc 0x558e81122148 2019-12-07T19:33:36.547+0000 7fe2ed47bbc0 10 _load_class rgw_gc from /usr/lib64/rados-classes/libcls_rgw_gc.so 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_class rgw_gc status 3 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method rgw_gc.rgw_gc_queue_init flags 3 0x7fe2d9e1a790 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method rgw_gc.rgw_gc_queue_enqueue flags 3 0x7fe2d9e1ae60 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method rgw_gc.rgw_gc_queue_list_entries flags 1 0x7fe2d9e1bf70 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method rgw_gc.rgw_gc_queue_remove_entries flags 3 0x7fe2d9e21950 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method rgw_gc.rgw_gc_queue_update_entry flags 3 0x7fe2d9e1eba0 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 _load_class rgw_gc success 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 open_all_classes found cephfs 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 _get_class adding new class name cephfs 0x558e81122268 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 _load_class cephfs from /usr/lib64/rados-classes/libcls_cephfs.so 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 0 /home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/15.0.0-8038-g503c226/rpm/el7/BUILD/ceph-15.0.0-8038-g503c226/src/cls/cephfs/cls_cephfs.cc:198: loading cephfs 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_class cephfs status 3 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method cephfs.accumulate_inode_metadata flags 3 0x7fe2d9bf9900 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 _load_class cephfs success 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 open_all_classes found numops 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 _get_class adding new class name numops 0x558e811224a8 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 _load_class numops from /usr/lib64/rados-classes/libcls_numops.so 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_class numops status 3 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method numops.add flags 3 0x7fe2d99e3ba0 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method numops.mul flags 3 0x7fe2d99e3170 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 _load_class numops success 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 open_all_classes found lua 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 0 _get_class not permitted to load lua 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 open_all_classes found queue 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 0 _get_class not permitted to load queue 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 open_all_classes found lock 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 _get_class adding new class name lock 0x558e811225c8 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 _load_class lock from /usr/lib64/rados-classes/libcls_lock.so 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_class lock status 3 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method lock.lock flags 11 0x7fe2d97c89c0 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method lock.unlock flags 11 0x7fe2d97c60a0 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method lock.break_lock flags 3 0x7fe2d97c58d0 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method lock.get_info flags 1 0x7fe2d97c7a50 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method lock.list_locks flags 1 0x7fe2d97c3b00 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method lock.assert_locked flags 9 0x7fe2d97c4a50 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method lock.set_cookie flags 11 0x7fe2d97c6740 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 _load_class lock success 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 open_all_classes found timeindex 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 _get_class adding new class name timeindex 0x558e81122808 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 _load_class timeindex from /usr/lib64/rados-classes/libcls_timeindex.so 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_class timeindex status 3 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method timeindex.add flags 3 0x7fe2d95ac080 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method timeindex.list flags 1 0x7fe2d95aafe0 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 register_cxx_method timeindex.trim flags 3 0x7fe2d95aa6b0 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 _load_class timeindex success 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 0 osd.0 0 crush map has features 288232575208783872, adjusting msgr requires for clients 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 0 osd.0 0 crush map has features 288232575208783872 was 8705, adjusting msgr requires for mons 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 0 osd.0 0 crush map has features 288232575208783872, adjusting msgr requires for osds 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 0 osd.0 0 check_osdmap_features enabling on-disk ERASURE CODES compat feature 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 osd.0 0 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e0 [0,0] lci=[0,0]) 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e8106c300 seq 3 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x233 (563) bytes fadvise_flags 0x0 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 _rm buffer(0x558e80229ef0 space 0x558e81078638 0x0~1000 clean) 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x233) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore.blob(0x558e810784d0) put_ref 0x0~233 Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x481) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c300 0 blobs 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~233: 0x0~233 Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x24e 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c300 cost 4803 (1 ios * 4000 + 803 bytes) 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c300 onodes 0x558e80251b80 shared_blobs 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c300 allocated 0x[] released 0x[] 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 prepare 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c300 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 io_done 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 osd.0 0 create_recoverystate_perf 2019-12-07T19:33:36.548+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:36.548+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:36.548+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4323 < min_flush_size 524288 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 10 osd.0 0 clear_temp_objects 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 0 osd.0 0 load_pgs 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:46e858f0:::pg_num_history:head# 0x0~0 2019-12-07T19:33:36.548+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:46e858f0:::pg_num_history:head# key 0x7f7fffffffffffffff46e858f0217067'_num_history!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:46e858f0:::pg_num_history:head# 0x0~0 = -2 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 20 osd.0 0 load_pgs pg_num_history pg_num_history(e0 pg_nums {} deleted_pools ) 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 10 osd.0 0 load_pgs ignoring unrecognized meta 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 0 osd.0 0 load_pgs opened 0 pgs 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 2 osd.0 0 superblock: I am osd.0 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 10 osd.0 0 create_logger 2019-12-07T19:33:36.549+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:33:36.549+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4346 < min_flush_size 524288 2019-12-07T19:33:36.549+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4346 < min_flush_size 524288 2019-12-07T19:33:36.549+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.549+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x123~10fa to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.549+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x123 0x123~10fa to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.549+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:36.549+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x123 2019-12-07T19:33:36.549+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x123 2019-12-07T19:33:36.549+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:36.549+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x21d and padding block with 0xde3 2019-12-07T19:33:36.549+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x121d 2019-12-07T19:33:36.549+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:36.549+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:36.549+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:36.549+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:36.549+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000229855s (0.000000092s flush + 0.000229763s kv commit) 2019-12-07T19:33:36.549+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:36.549+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:36.549+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8106c300> 2019-12-07T19:33:36.549+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:36.549+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 kv_submitted 2019-12-07T19:33:36.549+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c300 2019-12-07T19:33:36.549+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e8106c300 osr 0x558e8107a500 2019-12-07T19:33:36.549+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e8107b2c0) prepare_write seq 1 0x2000~1000 crc 27fad00c 2019-12-07T19:33:36.549+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 1 Processor -- start 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 -1 osd.0 0 log_to_monitors {default=true} 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 1 Processor -- start 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 1 Processor -- start 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 1 Processor -- start 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 1 Processor -- start 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 1 Processor -- start 2019-12-07T19:33:36.549+0000 7fe2ed47bbc0 1 Processor -- start 2019-12-07T19:33:36.549+0000 7fe2cc26f700 10 osd.0 0 agent_entry start 2019-12-07T19:33:36.549+0000 7fe2cc26f700 20 osd.0 0 agent_entry empty queue 2019-12-07T19:33:36.549+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:33:36.549+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:36.549+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:36.549+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:36.549+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:36.549+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:36.550+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:36.550+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:36.550+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:33:36.550+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:36.550+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:36.550+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:36.550+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:36.550+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:36.550+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:36.550+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:36.550+0000 7fe2ed47bbc0 1 --2- [v2:0.0.0.0:6818/37079,v1:0.0.0.0:6820/37079] >> v2:172.21.15.109:3300/0 conn(0x558e80f39800 0x558e80ef8b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:36.550+0000 7fe2c3a5e700 5 osd.0 0 heartbeat osd_stat(store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [] op hist []) 2019-12-07T19:33:36.550+0000 7fe2ed47bbc0 1 --2- [v2:0.0.0.0:6818/37079,v1:0.0.0.0:6820/37079] >> v2:172.21.15.131:3300/0 conn(0x558e80f39400 0x558e80ef9080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:36.550+0000 7fe2c3a5e700 20 osd.0 0 check_full_status cur ratio 1.27157e-07, physical ratio 1.27157e-07, new state none 2019-12-07T19:33:36.550+0000 7fe2ed47bbc0 1 --2- [v2:0.0.0.0:6818/37079,v1:0.0.0.0:6820/37079] >> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] conn(0x558e80f39000 0x558e80ef9600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:36.550+0000 7fe2ed47bbc0 1 -- [v2:0.0.0.0:6818/37079,v1:0.0.0.0:6820/37079] --> v2:172.21.15.109:3300/0 -- mon_getmap magic: 0 v1 -- 0x558e8022ee00 con 0x558e80f39800 2019-12-07T19:33:36.550+0000 7fe2ed47bbc0 1 -- [v2:0.0.0.0:6818/37079,v1:0.0.0.0:6820/37079] --> v2:172.21.15.131:3300/0 -- mon_getmap magic: 0 v1 -- 0x558e8022efc0 con 0x558e80f39400 2019-12-07T19:33:36.550+0000 7fe2ed47bbc0 1 -- [v2:0.0.0.0:6818/37079,v1:0.0.0.0:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- mon_getmap magic: 0 v1 -- 0x558e8022f340 con 0x558e80f39000 2019-12-07T19:33:36.550+0000 7fe2e7d3c700 1 --2- [v2:0.0.0.0:6818/37079,v1:0.0.0.0:6820/37079] >> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] conn(0x558e80f39000 0x558e80ef9600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:36.550+0000 7fe2e6d3a700 1 --2- [v2:0.0.0.0:6818/37079,v1:0.0.0.0:6820/37079] >> v2:172.21.15.131:3300/0 conn(0x558e80f39400 0x558e80ef9080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:36.550+0000 7fe2e7d3c700 1 --2- [v2:0.0.0.0:6818/37079,v1:0.0.0.0:6820/37079] >> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] conn(0x558e80f39000 0x558e80ef9600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).handle_hello peer v2:172.21.15.131:3301/0 says I am v2:172.21.15.131:39634/0 (socket says 172.21.15.131:39634) 2019-12-07T19:33:36.550+0000 7fe2e6d3a700 1 --2- [v2:0.0.0.0:6818/37079,v1:0.0.0.0:6820/37079] >> v2:172.21.15.131:3300/0 conn(0x558e80f39400 0x558e80ef9080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).handle_hello peer v2:172.21.15.131:3300/0 says I am v2:172.21.15.131:35962/0 (socket says 172.21.15.131:35962) 2019-12-07T19:33:36.550+0000 7fe2e753b700 1 --2- [v2:0.0.0.0:6818/37079,v1:0.0.0.0:6820/37079] >> v2:172.21.15.109:3300/0 conn(0x558e80f39800 0x558e80ef8b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:36.550+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] learned_addr learned my addr [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] (peer_addr_for_me v2:172.21.15.131:0/0) 2019-12-07T19:33:36.550+0000 7fe2e7d3c700 10 osd.0 0 new session (outgoing) 0x558e80245380 con=0x558e802a5000 addr=v2:172.21.15.131:6818/37079 2019-12-07T19:33:36.550+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> v2:172.21.15.131:3300/0 conn(0x558e80f39400 msgr2=0x558e80ef9080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2019-12-07T19:33:36.550+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> v2:172.21.15.131:3300/0 conn(0x558e80f39400 0x558e80ef9080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:36.550+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> v2:172.21.15.109:3300/0 conn(0x558e80f39800 msgr2=0x558e80ef8b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2019-12-07T19:33:36.550+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> v2:172.21.15.109:3300/0 conn(0x558e80f39800 0x558e80ef8b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:36.550+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558e810fd800 con 0x558e80f39000 2019-12-07T19:33:36.550+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> v2:172.21.15.131:3300/0 conn(0x558e80f39400 0x558e80ef9080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rx=0 tx=0).handle_auth_done state changed! 2019-12-07T19:33:36.550+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] conn(0x558e80f39000 0x558e80ef9600 crc :-1 s=READY pgs=16 cs=0 l=1 rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:36.550+0000 7fe2d959f700 10 osd.0 0 ms_handle_connect con 0x558e80f39000 2019-12-07T19:33:36.550+0000 7fe2d959f700 10 osd.0 0 ms_handle_connect on mon 2019-12-07T19:33:36.550+0000 7fe2d959f700 10 osd.0 0 requeue_pg_temp 0 + 0 -> 0 2019-12-07T19:33:36.550+0000 7fe2d959f700 20 osd.0 0 _send_ready_to_merge ready_to_merge_source {} not_ready_to_merge_source ready_to_merge_target {} not_ready_to_merge_target {} sent_ready_to_merge_source 2019-12-07T19:33:36.550+0000 7fe2d959f700 20 osd.0 0 send_pg_created 2019-12-07T19:33:36.550+0000 7fe2d959f700 10 osd.0 0 requeue_failures 0 + 0 -> 0 2019-12-07T19:33:36.550+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 1 ==== mon_map magic: 0 v1 ==== 383+0+0 (crc 0 0 0) 0x558e8021ee00 con 0x558e80f39000 2019-12-07T19:33:36.550+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- auth(proto 2 2 bytes epoch 0) v1 -- 0x558e8115a6c0 con 0x558e80f39000 2019-12-07T19:33:36.550+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (crc 0 0 0) 0x558e80231860 con 0x558e80f39000 2019-12-07T19:33:36.551+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 3 ==== mon_map magic: 0 v1 ==== 383+0+0 (crc 0 0 0) 0x558e810fd800 con 0x558e80f39000 2019-12-07T19:33:36.551+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (crc 0 0 0) 0x558e8115a6c0 con 0x558e80f39000 2019-12-07T19:33:36.551+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:36.551+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:36.551+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) collect_metadata 2019-12-07T19:33:36.551+0000 7fe2ed47bbc0 10 bluestore(/var/lib/ceph/osd/ceph-0) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:36.551+0000 7fe2ed47bbc0 1 bluestore(/var/lib/ceph/osd/ceph-0) collect_metadata devices span numa nodes 0 2019-12-07T19:33:36.551+0000 7fe2ed47bbc0 10 osd.0 0 mon_cmd_maybe_osd_create cmd: {"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} 2019-12-07T19:33:36.551+0000 7fe2ed47bbc0 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- mon_command({"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} v 0) v1 -- 0x558e810fda00 con 0x558e80f39000 2019-12-07T19:33:36.594+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.645+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.695+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.745+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.795+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.845+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.895+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.945+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:36.995+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.045+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.096+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.146+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.196+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.246+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.296+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.346+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.396+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.447+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.497+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845352040 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.513+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 5 ==== mon_command_ack([{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]=0 set osd(s) 0 to class 'ssd' v11) v1 ==== 131+0+0 (crc 0 0 0) 0x558e8118ea00 con 0x558e80f39000 2019-12-07T19:33:37.513+0000 7fe2ed47bbc0 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:37.513+0000 7fe2ed47bbc0 10 osd.0 0 update_crush_location crush location is "host=smithi131", "root=default" 2019-12-07T19:33:37.513+0000 7fe2ed47bbc0 10 osd.0 0 mon_cmd_maybe_osd_create cmd: {"prefix": "osd crush create-or-move", "id": 0, "weight":0.0879, "args": ["host=smithi131", "root=default"]} 2019-12-07T19:33:37.513+0000 7fe2ed47bbc0 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- mon_command({"prefix": "osd crush create-or-move", "id": 0, "weight":0.0879, "args": ["host=smithi131", "root=default"]} v 0) v1 -- 0x558e810fdc00 con 0x558e80f39000 2019-12-07T19:33:37.547+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3502 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 osd.0 0 tick 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 osd.0 0 do_waiters -- start 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 osd.0 0 do_waiters -- finish 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 osd.0 0 tick last_purged_snaps_scrub 0.000000 next 60.000000 ... now 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 osd.0 0 scrub_purged_snaps 2019-12-07T19:33:37.557+0000 7fe2e4535700 0 log_channel(cluster) log [DBG] : purged_snaps scrub starts 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 snap_mapper.run 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore.onode(0x558e8125c500).flush flush done 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 0 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 0 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 snap_mapper.run end, found 0 stray 2019-12-07T19:33:37.557+0000 7fe2e4535700 0 log_channel(cluster) log [DBG] : purged_snaps scrub ok 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 osd.0 0 scrub_purged_snaps done queueing pgs, updating superblock 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 osd.0 0 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e0 [0,0] lci=[0,0]) 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e8106c900 seq 4 2019-12-07T19:33:37.557+0000 7fe2e4535700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c900 0 blobs 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c900 cost 4803 (1 ios * 4000 + 803 bytes) 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c900 onodes 0x558e80251b80 shared_blobs 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c900 allocated 0x[] released 0x[] 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 prepare 2019-12-07T19:33:37.557+0000 7fe2e4535700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c900 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 io_done 2019-12-07T19:33:37.557+0000 7fe2e4535700 10 osd.0 0 scrub_purged_snaps done 2019-12-07T19:33:37.557+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:37.557+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:37.557+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4270 < min_flush_size 524288 2019-12-07T19:33:37.557+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:33:37.557+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:37.557+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959999), bluefs_ratio 0.0399535 2019-12-07T19:33:37.557+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4293 < min_flush_size 524288 2019-12-07T19:33:37.557+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4293 < min_flush_size 524288 2019-12-07T19:33:37.557+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:37.557+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x121d~10c5 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:37.557+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x121d 0x121d~10c5 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:37.557+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:37.557+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x121d 2019-12-07T19:33:37.557+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x21d 2019-12-07T19:33:37.557+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:37.557+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x2e2 and padding block with 0xd1e 2019-12-07T19:33:37.557+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x22e2 2019-12-07T19:33:37.557+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:37.557+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:37.557+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:37.557+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:37.557+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000150384s (0.000000101s flush + 0.000150283s kv commit) 2019-12-07T19:33:37.557+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:37.557+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:37.557+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8106c900> 2019-12-07T19:33:37.557+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:37.557+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 kv_submitted 2019-12-07T19:33:37.557+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c900 2019-12-07T19:33:37.557+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e8106c900 osr 0x558e8107a500 2019-12-07T19:33:37.557+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e8107b2c0) _discard drop 1 0x2000~1000 2019-12-07T19:33:37.557+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e8107b2c0) prepare_write seq 2 0x2000~1000 crc 902c5272 2019-12-07T19:33:37.557+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:37.590+0000 7fe2e2ca2700 10 osd.0 0 tick_without_osd_lock 2019-12-07T19:33:37.590+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:37.597+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.647+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.697+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.747+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.797+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.848+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.898+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.948+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:37.998+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.048+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.098+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.148+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.198+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.249+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.299+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.349+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.399+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.449+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.499+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415009 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 3518 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.516+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 6 ==== mon_command_ack([{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0879, "args": ["host=smithi131", "root=default"]}]=0 create-or-move updating item name 'osd.0' weight 0.0879 at location {host=smithi131,root=default} to crush map v12) v1 ==== 252+0+0 (crc 0 0 0) 0x558e8118e800 con 0x558e80f39000 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 10 osd.0 0 ensuring pgs have consumed prior maps 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 7 osd.0 0 consume_map version 0 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 20 osd.0 0 prune_pg_created 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 10 osd.0:0.consume_map 0 (was 0) 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 10 osd.0:1.consume_map 0 (was 0) 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 10 osd.0:2.consume_map 0 (was 0) 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 10 osd.0:3.consume_map 0 (was 0) 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 10 osd.0:4.consume_map 0 (was 0) 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 10 osd.0:5.consume_map 0 (was 0) 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 10 osd.0:6.consume_map 0 (was 0) 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 10 osd.0:7.consume_map 0 (was 0) 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 10 osd.0 0 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 0 osd.0 0 done with init, starting boot process 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- mon_subscribe({mgrmap=0+,osd_pg_creates=0+}) v3 -- 0x558e810fde00 con 0x558e80f39000 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 1 osd.0 0 start_boot 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 10 osd.0 0 start_boot - have maps 0..0 2019-12-07T19:33:38.516+0000 7fe2ed47bbc0 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x558e8022f180 con 0x558e80f39000 2019-12-07T19:33:38.519+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 7 ==== mgrmap(e 4) v1 ==== 45131+0+0 (crc 0 0 0) 0x558e8106c000 con 0x558e80f39000 2019-12-07T19:33:38.519+0000 7fe2d959f700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:38.520+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgropen(unknown.0) v3 -- 0x558e8027f600 con 0x558e81748000 2019-12-07T19:33:38.520+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:38.521+0000 7fe2d959f700 10 osd.0 0 ms_handle_connect con 0x558e81748000 2019-12-07T19:33:38.521+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 crc :-1 s=READY pgs=25 cs=0 l=1 rx=0 tx=0).ready entity=mgr.4099 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:38.521+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 msgr2=0x558e80efa680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 61 2019-12-07T19:33:38.521+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mgr.4099 v2:172.21.15.131:6800/36137 1 ==== mgrconfigure(period=5, threshold=5) v3 ==== 12+0+0 (crc 0 0 0) 0x558e8021f200 con 0x558e81748000 2019-12-07T19:33:38.521+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 msgr2=0x558e80efa680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:38.521+0000 7fe2d959f700 10 osd.0 0 setting 0 queries 2019-12-07T19:33:38.521+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 crc :-1 s=READY pgs=25 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:38.521+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 crc :-1 s=READY pgs=25 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:38.521+0000 7fe2d959f700 20 osd.0 0 reports for 0 queries 2019-12-07T19:33:38.521+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.0 +109-0 packed 1198 daemon_metrics=2) v8 -- 0x558e81174380 con 0x558e81748000 2019-12-07T19:33:38.521+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x558e81174380 con 0x558e81748000 2019-12-07T19:33:38.521+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 msgr2=0x558e80efa680 unknown :-1 s=STATE_CLOSED l=1).mark_down 2019-12-07T19:33:38.521+0000 7fe2d959f700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:38.523+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 8 ==== mon_get_version_reply(handle=1 version=12) v2 ==== 24+0+0 (crc 0 0 0) 0x558e80231680 con 0x558e80f39000 2019-12-07T19:33:38.523+0000 7fe2d5a82700 10 osd.0 0 _preboot _preboot mon has osdmaps 1..12 2019-12-07T19:33:38.523+0000 7fe2d5a82700 5 osd.0 0 heartbeat osd_stat(store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [] op hist []) 2019-12-07T19:33:38.523+0000 7fe2d5a82700 20 osd.0 0 check_full_status cur ratio 1.27157e-07, physical ratio 1.27157e-07, new state none 2019-12-07T19:33:38.523+0000 7fe2d5a82700 -1 osd.0 0 waiting for initial osdmap 2019-12-07T19:33:38.523+0000 7fe2d5a82700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- mon_subscribe({osdmap=1}) v3 -- 0x558e8021f200 con 0x558e80f39000 2019-12-07T19:33:38.527+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 9 ==== osd_map(1..12 src has 1..12) v4 ==== 5778+0+0 (crc 0 0 0) 0x558e8115a480 con 0x558e80f39000 2019-12-07T19:33:38.527+0000 7fe2d959f700 20 osd.0 0 OSD::ms_dispatch: osd_map(1..12 src has 1..12) v4 2019-12-07T19:33:38.527+0000 7fe2d959f700 10 osd.0 0 do_waiters -- start 2019-12-07T19:33:38.527+0000 7fe2d959f700 10 osd.0 0 do_waiters -- finish 2019-12-07T19:33:38.527+0000 7fe2d959f700 20 osd.0 0 _dispatch 0x558e8115a480 osd_map(1..12 src has 1..12) v4 2019-12-07T19:33:38.527+0000 7fe2d959f700 3 osd.0 0 handle_osd_map epochs [1,12], i have 0, src has [1,12] 2019-12-07T19:33:38.527+0000 7fe2d959f700 10 osd.0 0 handle_osd_map got inc map for epoch 1 2019-12-07T19:33:38.527+0000 7fe2d959f700 20 osd.0 0 got_full_map 1, nothing requested 2019-12-07T19:33:38.527+0000 7fe2d959f700 10 osd.0 0 handle_osd_map got inc map for epoch 2 2019-12-07T19:33:38.527+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:8d9276bf:::osdmap.1:0# 0x0~0 2019-12-07T19:33:38.527+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:8d9276bf:::osdmap.1:0# key 0x7f7fffffffffffffff8d9276bf216f73'dmap.1!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.527+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.527+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:8d9276bf:::osdmap.1:0# 0x0~0 = -2 2019-12-07T19:33:38.527+0000 7fe2d959f700 20 osd.0 0 got_full_map 2, nothing requested 2019-12-07T19:33:38.527+0000 7fe2d959f700 10 osd.0 0 handle_osd_map got inc map for epoch 3 2019-12-07T19:33:38.527+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:829276bf:::osdmap.2:0# 0x0~0 2019-12-07T19:33:38.527+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:829276bf:::osdmap.2:0# key 0x7f7fffffffffffffff829276bf216f73'dmap.2!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.527+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.527+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:829276bf:::osdmap.2:0# 0x0~0 = -2 2019-12-07T19:33:38.527+0000 7fe2d959f700 20 osd.0 0 got_full_map 3, nothing requested 2019-12-07T19:33:38.527+0000 7fe2d959f700 10 osd.0 0 handle_osd_map got inc map for epoch 4 2019-12-07T19:33:38.527+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:887276bf:::osdmap.3:0# 0x0~0 2019-12-07T19:33:38.527+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:887276bf:::osdmap.3:0# key 0x7f7fffffffffffffff887276bf216f73'dmap.3!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.527+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.527+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:887276bf:::osdmap.3:0# 0x0~0 = -2 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 osd.0 0 got_full_map 4, nothing requested 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 osd.0 0 handle_osd_map got inc map for epoch 5 2019-12-07T19:33:38.528+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:85f276bf:::osdmap.4:0# 0x0~0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:85f276bf:::osdmap.4:0# key 0x7f7fffffffffffffff85f276bf216f73'dmap.4!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:85f276bf:::osdmap.4:0# 0x0~0 = -2 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 osd.0 0 got_full_map 5, nothing requested 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 osd.0 0 handle_osd_map got inc map for epoch 6 2019-12-07T19:33:38.528+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:8ef276bf:::osdmap.5:0# 0x0~0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:8ef276bf:::osdmap.5:0# key 0x7f7fffffffffffffff8ef276bf216f73'dmap.5!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:8ef276bf:::osdmap.5:0# 0x0~0 = -2 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 osd.0 0 got_full_map 6, nothing requested 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 osd.0 0 handle_osd_map got inc map for epoch 7 2019-12-07T19:33:38.528+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:803276bf:::osdmap.6:0# 0x0~0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:803276bf:::osdmap.6:0# key 0x7f7fffffffffffffff803276bf216f73'dmap.6!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:803276bf:::osdmap.6:0# 0x0~0 = -2 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 osd.0 0 got_full_map 7, nothing requested 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 osd.0 0 handle_osd_map got inc map for epoch 8 2019-12-07T19:33:38.528+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:8bb276bf:::osdmap.7:0# 0x0~0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:8bb276bf:::osdmap.7:0# key 0x7f7fffffffffffffff8bb276bf216f73'dmap.7!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:8bb276bf:::osdmap.7:0# 0x0~0 = -2 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 osd.0 0 got_full_map 8, nothing requested 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 osd.0 0 handle_osd_map got inc map for epoch 9 2019-12-07T19:33:38.528+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:86b276bf:::osdmap.8:0# 0x0~0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:86b276bf:::osdmap.8:0# key 0x7f7fffffffffffffff86b276bf216f73'dmap.8!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:86b276bf:::osdmap.8:0# 0x0~0 = -2 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 osd.0 0 got_full_map 9, nothing requested 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 osd.0 0 handle_osd_map got inc map for epoch 10 2019-12-07T19:33:38.528+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:8c4276bf:::osdmap.9:0# 0x0~0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:8c4276bf:::osdmap.9:0# key 0x7f7fffffffffffffff8c4276bf216f73'dmap.9!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:8c4276bf:::osdmap.9:0# 0x0~0 = -2 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 osd.0 0 got_full_map 10, nothing requested 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 osd.0 0 handle_osd_map got inc map for epoch 11 2019-12-07T19:33:38.528+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3890e826:::osdmap.10:0# 0x0~0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3890e826:::osdmap.10:0# key 0x7f7fffffffffffffff3890e8'&!osdmap.10!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3890e826:::osdmap.10:0# 0x0~0 = -2 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 osd.0 0 got_full_map 11, nothing requested 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 osd.0 0 handle_osd_map got inc map for epoch 12 2019-12-07T19:33:38.528+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3570e826:::osdmap.11:0# 0x0~0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3570e826:::osdmap.11:0# key 0x7f7fffffffffffffff3570e8'&!osdmap.11!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3570e826:::osdmap.11:0# 0x0~0 = -2 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 osd.0 0 got_full_map 12, nothing requested 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 osd.0 0 handle_osd_map pg_num_history pg_num_history(e12 pg_nums {} deleted_pools ) 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps purged_snaps {1={},2={},3={},4={},5={},6={},7={},8={},9={},10={},11={},12={}} 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 osd.0 0 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e12 [1,12] lci=[0,0]) 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e8106c000 seq 5 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:60c2fa6d:::inc_osdmap.1:0# key 0x7f7fffffffffffffff60c2fa'm!inc_osdmap.1!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:60c2fa6d:::inc_osdmap.1:0# 0x0~3b2 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1028 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:60c2fa6d:::inc_osdmap.1:0# 0x0~3b2 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~3b2 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xc4e on front/back, now 0x0~1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x3000~1000/1000,1000,0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x3000~1000] 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817880e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81788150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817880e0 blob([0x3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788150 sbid 0x0)) 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.blob(0x558e817880e0) get_ref 0x0~3b2 Blob(0x558e817880e0 blob([0x3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788150 sbid 0x0)) 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.blob(0x558e817880e0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~3b2: 0x0~3b2 Blob(0x558e817880e0 blob([0x3000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x3b2) SharedBlob(0x558e81788150 sbid 0x0)) 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81788168 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x3b2 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125d288) dirty_range mark inline shard dirty 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:60c2fa6d:::inc_osdmap.1:0# 0x0~3b2 = 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:8d9276bf:::osdmap.1:0# key 0x7f7fffffffffffffff8d9276bf216f73'dmap.1!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:8d9276bf:::osdmap.1:0# 0x0~2c6 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1029 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:8d9276bf:::osdmap.1:0# 0x0~2c6 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~2c6 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xd3a on front/back, now 0x0~1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x4000~1000/1000,1000,0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x4000~1000] 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81788540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817884d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81788540 blob([0x4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817884d0 sbid 0x0)) 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.blob(0x558e81788540) get_ref 0x0~2c6 Blob(0x558e81788540 blob([0x4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817884d0 sbid 0x0)) 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.blob(0x558e81788540) get_ref init 0x1000, 1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2c6: 0x0~2c6 Blob(0x558e81788540 blob([0x4000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x2c6) SharedBlob(0x558e817884d0 sbid 0x0)) 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e817884e8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x2c6 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125d508) dirty_range mark inline shard dirty 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:8d9276bf:::osdmap.1:0# 0x0~2c6 = 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:6b02fa6d:::inc_osdmap.2:0# key 0x7f7fffffffffffffff6b02fa'm!inc_osdmap.2!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:6b02fa6d:::inc_osdmap.2:0# 0x0~24c 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1030 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:6b02fa6d:::inc_osdmap.2:0# 0x0~24c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24c 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xdb4 on front/back, now 0x0~1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x5000~1000/1000,1000,0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x5000~1000] 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817883f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81788380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817883f0 blob([0x5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788380 sbid 0x0)) 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.blob(0x558e817883f0) get_ref 0x0~24c Blob(0x558e817883f0 blob([0x5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788380 sbid 0x0)) 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.blob(0x558e817883f0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~24c: 0x0~24c Blob(0x558e817883f0 blob([0x5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24c) SharedBlob(0x558e81788380 sbid 0x0)) 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81788398 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x24c 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125d788) dirty_range mark inline shard dirty 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:6b02fa6d:::inc_osdmap.2:0# 0x0~24c = 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:829276bf:::osdmap.2:0# key 0x7f7fffffffffffffff829276bf216f73'dmap.2!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.528+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:829276bf:::osdmap.2:0# 0x0~2c6 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1031 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:829276bf:::osdmap.2:0# 0x0~2c6 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~2c6 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xd3a on front/back, now 0x0~1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.528+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x6000~1000/1000,1000,0 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x6000~1000] 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817882a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81788000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817882a0 blob([0x6000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788000 sbid 0x0)) 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.blob(0x558e817882a0) get_ref 0x0~2c6 Blob(0x558e817882a0 blob([0x6000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788000 sbid 0x0)) 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.blob(0x558e817882a0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2c6: 0x0~2c6 Blob(0x558e817882a0 blob([0x6000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x2c6) SharedBlob(0x558e81788000 sbid 0x0)) 2019-12-07T19:33:38.528+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81788018 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x2c6 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125da08) dirty_range mark inline shard dirty 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:829276bf:::osdmap.2:0# 0x0~2c6 = 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:6602fa6d:::inc_osdmap.3:0# key 0x7f7fffffffffffffff6602fa'm!inc_osdmap.3!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:6602fa6d:::inc_osdmap.3:0# 0x0~108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1032 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:6602fa6d:::inc_osdmap.3:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x7000~1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x7000~1000] 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e811d1ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d1e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e811d1ea0 blob([0x7000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d1e30 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e811d1ea0) get_ref 0x0~108 Blob(0x558e811d1ea0 blob([0x7000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d1e30 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e811d1ea0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558e811d1ea0 blob([0x7000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558e811d1e30 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e811d1e48 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125dc88) dirty_range mark inline shard dirty 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:6602fa6d:::inc_osdmap.3:0# 0x0~108 = 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:887276bf:::osdmap.3:0# key 0x7f7fffffffffffffff887276bf216f73'dmap.3!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:887276bf:::osdmap.3:0# 0x0~339 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1033 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:887276bf:::osdmap.3:0# 0x0~339 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~339 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xcc7 on front/back, now 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x8000~1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x8000~1000] 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e811d1f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d1d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e811d1f10 blob([0x8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d1d50 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e811d1f10) get_ref 0x0~339 Blob(0x558e811d1f10 blob([0x8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d1d50 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e811d1f10) get_ref init 0x1000, 1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~339: 0x0~339 Blob(0x558e811d1f10 blob([0x8000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x339) SharedBlob(0x558e811d1d50 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e811d1d68 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x339 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796108) dirty_range mark inline shard dirty 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:887276bf:::osdmap.3:0# 0x0~339 = 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:6c82fa6d:::inc_osdmap.4:0# key 0x7f7fffffffffffffff6c82fa'm!inc_osdmap.4!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:6c82fa6d:::inc_osdmap.4:0# 0x0~108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1034 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:6c82fa6d:::inc_osdmap.4:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x9000~1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x9000~1000] 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817885b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81788620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817885b0 blob([0x9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788620 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e817885b0) get_ref 0x0~108 Blob(0x558e817885b0 blob([0x9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788620 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e817885b0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558e817885b0 blob([0x9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558e81788620 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81788638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796388) dirty_range mark inline shard dirty 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:6c82fa6d:::inc_osdmap.4:0# 0x0~108 = 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:85f276bf:::osdmap.4:0# key 0x7f7fffffffffffffff85f276bf216f73'dmap.4!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:85f276bf:::osdmap.4:0# 0x0~3ac 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1035 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:85f276bf:::osdmap.4:0# 0x0~3ac - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~3ac 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xc54 on front/back, now 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0xa000~1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0xa000~1000] 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81788700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81788770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81788700 blob([0xa000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788770 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81788700) get_ref 0x0~3ac Blob(0x558e81788700 blob([0xa000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788770 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81788700) get_ref init 0x1000, 1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~3ac: 0x0~3ac Blob(0x558e81788700 blob([0xa000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x3ac) SharedBlob(0x558e81788770 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81788788 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x3ac 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796608) dirty_range mark inline shard dirty 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:85f276bf:::osdmap.4:0# 0x0~3ac = 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:6362fa6d:::inc_osdmap.5:0# key 0x7f7fffffffffffffff6362fa'm!inc_osdmap.5!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:6362fa6d:::inc_osdmap.5:0# 0x0~108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1036 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:6362fa6d:::inc_osdmap.5:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0xb000~1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0xb000~1000] 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81788850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817888c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81788850 blob([0xb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817888c0 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81788850) get_ref 0x0~108 Blob(0x558e81788850 blob([0xb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817888c0 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81788850) get_ref init 0x1000, 1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558e81788850 blob([0xb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558e817888c0 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e817888d8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796888) dirty_range mark inline shard dirty 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:6362fa6d:::inc_osdmap.5:0# 0x0~108 = 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:8ef276bf:::osdmap.5:0# key 0x7f7fffffffffffffff8ef276bf216f73'dmap.5!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:8ef276bf:::osdmap.5:0# 0x0~41f 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1037 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:8ef276bf:::osdmap.5:0# 0x0~41f - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~41f 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbe1 on front/back, now 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0xc000~1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0xc000~1000] 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817889a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81788a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817889a0 blob([0xc000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788a10 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e817889a0) get_ref 0x0~41f Blob(0x558e817889a0 blob([0xc000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788a10 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e817889a0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~41f: 0x0~41f Blob(0x558e817889a0 blob([0xc000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x41f) SharedBlob(0x558e81788a10 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81788a28 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x41f 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796b08) dirty_range mark inline shard dirty 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:8ef276bf:::osdmap.5:0# 0x0~41f = 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:69e2fa6d:::inc_osdmap.6:0# key 0x7f7fffffffffffffff69e2fa'm!inc_osdmap.6!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:69e2fa6d:::inc_osdmap.6:0# 0x0~108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1038 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:69e2fa6d:::inc_osdmap.6:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0xd000~1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0xd000~1000] 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81788af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81788b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81788af0 blob([0xd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788b60 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81788af0) get_ref 0x0~108 Blob(0x558e81788af0 blob([0xd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788b60 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81788af0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558e81788af0 blob([0xd000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558e81788b60 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81788b78 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796d88) dirty_range mark inline shard dirty 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:69e2fa6d:::inc_osdmap.6:0# 0x0~108 = 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:803276bf:::osdmap.6:0# key 0x7f7fffffffffffffff803276bf216f73'dmap.6!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:803276bf:::osdmap.6:0# 0x0~492 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1039 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:803276bf:::osdmap.6:0# 0x0~492 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~492 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xb6e on front/back, now 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0xe000~1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0xe000~1000] 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81788c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81788cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81788c40 blob([0xe000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788cb0 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81788c40) get_ref 0x0~492 Blob(0x558e81788c40 blob([0xe000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788cb0 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81788c40) get_ref init 0x1000, 1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~492: 0x0~492 Blob(0x558e81788c40 blob([0xe000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x492) SharedBlob(0x558e81788cb0 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81788cc8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x492 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797008) dirty_range mark inline shard dirty 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:803276bf:::osdmap.6:0# 0x0~492 = 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:64e2fa6d:::inc_osdmap.7:0# key 0x7f7fffffffffffffff64e2fa'm!inc_osdmap.7!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:64e2fa6d:::inc_osdmap.7:0# 0x0~108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1040 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:64e2fa6d:::inc_osdmap.7:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0xf000~1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0xf000~1000] 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81788d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81788e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81788d90 blob([0xf000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788e00 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81788d90) get_ref 0x0~108 Blob(0x558e81788d90 blob([0xf000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788e00 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81788d90) get_ref init 0x1000, 1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558e81788d90 blob([0xf000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558e81788e00 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81788e18 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797288) dirty_range mark inline shard dirty 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:64e2fa6d:::inc_osdmap.7:0# 0x0~108 = 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:8bb276bf:::osdmap.7:0# key 0x7f7fffffffffffffff8bb276bf216f73'dmap.7!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:8bb276bf:::osdmap.7:0# 0x0~505 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1041 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:8bb276bf:::osdmap.7:0# 0x0~505 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~505 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xafb on front/back, now 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x10000~1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x10000~1000] 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81788ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81788f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81788ee0 blob([0x10000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788f50 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81788ee0) get_ref 0x0~505 Blob(0x558e81788ee0 blob([0x10000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81788f50 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81788ee0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~505: 0x0~505 Blob(0x558e81788ee0 blob([0x10000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x505) SharedBlob(0x558e81788f50 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81788f68 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x505 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797508) dirty_range mark inline shard dirty 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:8bb276bf:::osdmap.7:0# 0x0~505 = 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:6f22fa6d:::inc_osdmap.8:0# key 0x7f7fffffffffffffff6f22fa'm!inc_osdmap.8!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:6f22fa6d:::inc_osdmap.8:0# 0x0~108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1042 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:6f22fa6d:::inc_osdmap.8:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x11000~1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x11000~1000] 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81789030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817890a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81789030 blob([0x11000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817890a0 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81789030) get_ref 0x0~108 Blob(0x558e81789030 blob([0x11000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817890a0 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81789030) get_ref init 0x1000, 1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558e81789030 blob([0x11000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558e817890a0 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e817890b8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797788) dirty_range mark inline shard dirty 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:6f22fa6d:::inc_osdmap.8:0# 0x0~108 = 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:86b276bf:::osdmap.8:0# key 0x7f7fffffffffffffff86b276bf216f73'dmap.8!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:86b276bf:::osdmap.8:0# 0x0~578 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1043 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:86b276bf:::osdmap.8:0# 0x0~578 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~578 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xa88 on front/back, now 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x12000~1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x12000~1000] 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81789180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817891f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81789180 blob([0x12000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817891f0 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81789180) get_ref 0x0~578 Blob(0x558e81789180 blob([0x12000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817891f0 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e81789180) get_ref init 0x1000, 1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~578: 0x0~578 Blob(0x558e81789180 blob([0x12000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x578) SharedBlob(0x558e817891f0 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789208 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x578 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797a08) dirty_range mark inline shard dirty 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:86b276bf:::osdmap.8:0# 0x0~578 = 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:61a2fa6d:::inc_osdmap.9:0# key 0x7f7fffffffffffffff61a2fa'm!inc_osdmap.9!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:61a2fa6d:::inc_osdmap.9:0# 0x0~108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1044 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:61a2fa6d:::inc_osdmap.9:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x13000~1000/1000,1000,0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x13000~1000] 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817892d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81789340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817892d0 blob([0x13000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789340 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e817892d0) get_ref 0x0~108 Blob(0x558e817892d0 blob([0x13000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789340 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.blob(0x558e817892d0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558e817892d0 blob([0x13000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558e81789340 sbid 0x0)) 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789358 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x108 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797c88) dirty_range mark inline shard dirty 2019-12-07T19:33:38.529+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:61a2fa6d:::inc_osdmap.9:0# 0x0~108 = 0 2019-12-07T19:33:38.529+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:8c4276bf:::osdmap.9:0# key 0x7f7fffffffffffffff8c4276bf216f73'dmap.9!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:8c4276bf:::osdmap.9:0# 0x0~5eb 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1045 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:8c4276bf:::osdmap.9:0# 0x0~5eb - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~5eb 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xa15 on front/back, now 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x14000~1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x14000~1000] 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81789420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81789490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81789420 blob([0x14000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789490 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e81789420) get_ref 0x0~5eb Blob(0x558e81789420 blob([0x14000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789490 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e81789420) get_ref init 0x1000, 1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~5eb: 0x0~5eb Blob(0x558e81789420 blob([0x14000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x5eb) SharedBlob(0x558e81789490 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e817894a8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x5eb 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8108) dirty_range mark inline shard dirty 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:8c4276bf:::osdmap.9:0# 0x0~5eb = 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c003972f:::inc_osdmap.10:0# key 0x7f7fffffffffffffffc00397'/!inc_osdmap.10!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c003972f:::inc_osdmap.10:0# 0x0~108 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1046 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c003972f:::inc_osdmap.10:0# 0x0~108 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~108 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xef8 on front/back, now 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x15000~1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x15000~1000] 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81789570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817895e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81789570 blob([0x15000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817895e0 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e81789570) get_ref 0x0~108 Blob(0x558e81789570 blob([0x15000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817895e0 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e81789570) get_ref init 0x1000, 1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~108: 0x0~108 Blob(0x558e81789570 blob([0x15000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x108) SharedBlob(0x558e817895e0 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e817895f8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x108 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8388) dirty_range mark inline shard dirty 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c003972f:::inc_osdmap.10:0# 0x0~108 = 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3890e826:::osdmap.10:0# key 0x7f7fffffffffffffff3890e8'&!osdmap.10!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3890e826:::osdmap.10:0# 0x0~65e 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1047 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3890e826:::osdmap.10:0# 0x0~65e - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~65e 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0x9a2 on front/back, now 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x16000~1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x16000~1000] 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817896c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81789730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817896c0 blob([0x16000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789730 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e817896c0) get_ref 0x0~65e Blob(0x558e817896c0 blob([0x16000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789730 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e817896c0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~65e: 0x0~65e Blob(0x558e817896c0 blob([0x16000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x65e) SharedBlob(0x558e81789730 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789748 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x65e 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8608) dirty_range mark inline shard dirty 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3890e826:::osdmap.10:0# 0x0~65e = 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:cb83972f:::inc_osdmap.11:0# key 0x7f7fffffffffffffffcb8397'/!inc_osdmap.11!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:cb83972f:::inc_osdmap.11:0# 0x0~33a 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1048 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:cb83972f:::inc_osdmap.11:0# 0x0~33a - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~33a 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xcc6 on front/back, now 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x17000~1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x17000~1000] 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81789810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81789880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81789810 blob([0x17000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789880 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e81789810) get_ref 0x0~33a Blob(0x558e81789810 blob([0x17000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789880 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e81789810) get_ref init 0x1000, 1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~33a: 0x0~33a Blob(0x558e81789810 blob([0x17000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x33a) SharedBlob(0x558e81789880 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789898 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x33a 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8888) dirty_range mark inline shard dirty 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:cb83972f:::inc_osdmap.11:0# 0x0~33a = 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3570e826:::osdmap.11:0# key 0x7f7fffffffffffffff3570e8'&!osdmap.11!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3570e826:::osdmap.11:0# 0x0~74c 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1049 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3570e826:::osdmap.11:0# 0x0~74c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~74c 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0x8b4 on front/back, now 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x18000~1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x18000~1000] 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81789960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817899d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81789960 blob([0x18000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817899d0 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e81789960) get_ref 0x0~74c Blob(0x558e81789960 blob([0x18000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817899d0 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e81789960) get_ref init 0x1000, 1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~74c: 0x0~74c Blob(0x558e81789960 blob([0x18000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x74c) SharedBlob(0x558e817899d0 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e817899e8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x74c 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8b08) dirty_range mark inline shard dirty 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3570e826:::osdmap.11:0# 0x0~74c = 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c683972f:::inc_osdmap.12:0# key 0x7f7fffffffffffffffc68397'/!inc_osdmap.12!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c683972f:::inc_osdmap.12:0# 0x0~496 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1050 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c683972f:::inc_osdmap.12:0# 0x0~496 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~496 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xb6a on front/back, now 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x19000~1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x19000~1000] 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81789ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81789b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81789ab0 blob([0x19000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789b20 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e81789ab0) get_ref 0x0~496 Blob(0x558e81789ab0 blob([0x19000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789b20 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e81789ab0) get_ref init 0x1000, 1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~496: 0x0~496 Blob(0x558e81789ab0 blob([0x19000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x496) SharedBlob(0x558e81789b20 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789b38 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x496 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8d88) dirty_range mark inline shard dirty 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c683972f:::inc_osdmap.12:0# 0x0~496 = 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3e70e826:::osdmap.12:0# key 0x7f7fffffffffffffff3e70e8'&!osdmap.12!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3e70e826:::osdmap.12:0# 0x0~8a8 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1051 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3e70e826:::osdmap.12:0# 0x0~8a8 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~8a8 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0x758 on front/back, now 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x1a000~1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x1a000~1000] 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81789c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81789c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81789c00 blob([0x1a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789c70 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e81789c00) get_ref 0x0~8a8 Blob(0x558e81789c00 blob([0x1a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789c70 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e81789c00) get_ref init 0x1000, 1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~8a8: 0x0~8a8 Blob(0x558e81789c00 blob([0x1a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x8a8) SharedBlob(0x558e81789c70 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789c88 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x8a8 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9008) dirty_range mark inline shard dirty 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3e70e826:::osdmap.12:0# 0x0~8a8 = 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:46e858f0:::pg_num_history:head# key 0x7f7fffffffffffffff46e858f0217067'_num_history!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1052 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~12 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xfee on front/back, now 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x1b000~1000/1000,1000,0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x1b000~1000] 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81789d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81789dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) get_ref init 0x1000, 1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~12: 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x12 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 0 blobs 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c000 cost 129493 (26 ios * 4000 + 25493 bytes) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c000 onodes 0x558e80251b80,0x558e8125c500,0x558e8125d180,0x558e8125d400,0x558e8125d680,0x558e8125d900,0x558e8125db80,0x558e81796000,0x558e81796280,0x558e81796500,0x558e81796780,0x558e81796a00,0x558e81796c80,0x558e81796f00,0x558e81797180,0x558e81797400,0x558e81797680,0x558e81797900,0x558e81797b80,0x558e817b8000,0x558e817b8280,0x558e817b8500,0x558e817b8780,0x558e817b8a00,0x558e817b8c80,0x558e817b8f00,0x558e817b9180 shared_blobs 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125c608) update #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c1a3fc6e:::purged_snaps:0# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125d288) update #-1:60c2fa6d:::inc_osdmap.1:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125d288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:60c2fa6d:::inc_osdmap.1:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125d508) update #-1:8d9276bf:::osdmap.1:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125d508) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:8d9276bf:::osdmap.1:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125d788) update #-1:6b02fa6d:::inc_osdmap.2:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125d788) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:6b02fa6d:::inc_osdmap.2:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125da08) update #-1:829276bf:::osdmap.2:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125da08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:829276bf:::osdmap.2:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125dc88) update #-1:6602fa6d:::inc_osdmap.3:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e8125dc88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:6602fa6d:::inc_osdmap.3:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796108) update #-1:887276bf:::osdmap.3:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796108) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:887276bf:::osdmap.3:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796388) update #-1:6c82fa6d:::inc_osdmap.4:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796388) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:6c82fa6d:::inc_osdmap.4:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796608) update #-1:85f276bf:::osdmap.4:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796608) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:85f276bf:::osdmap.4:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796888) update #-1:6362fa6d:::inc_osdmap.5:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796888) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:6362fa6d:::inc_osdmap.5:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796b08) update #-1:8ef276bf:::osdmap.5:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796b08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:8ef276bf:::osdmap.5:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796d88) update #-1:69e2fa6d:::inc_osdmap.6:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81796d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:69e2fa6d:::inc_osdmap.6:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797008) update #-1:803276bf:::osdmap.6:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797008) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:803276bf:::osdmap.6:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797288) update #-1:64e2fa6d:::inc_osdmap.7:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:64e2fa6d:::inc_osdmap.7:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797508) update #-1:8bb276bf:::osdmap.7:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797508) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:8bb276bf:::osdmap.7:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797788) update #-1:6f22fa6d:::inc_osdmap.8:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797788) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:6f22fa6d:::inc_osdmap.8:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797a08) update #-1:86b276bf:::osdmap.8:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:86b276bf:::osdmap.8:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797c88) update #-1:61a2fa6d:::inc_osdmap.9:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e81797c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:61a2fa6d:::inc_osdmap.9:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8108) update #-1:8c4276bf:::osdmap.9:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8108) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:8c4276bf:::osdmap.9:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8388) update #-1:c003972f:::inc_osdmap.10:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8388) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c003972f:::inc_osdmap.10:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8608) update #-1:3890e826:::osdmap.10:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8608) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3890e826:::osdmap.10:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8888) update #-1:cb83972f:::inc_osdmap.11:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8888) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:cb83972f:::inc_osdmap.11:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8b08) update #-1:3570e826:::osdmap.11:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8b08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3570e826:::osdmap.11:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8d88) update #-1:c683972f:::inc_osdmap.12:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b8d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c683972f:::inc_osdmap.12:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9008) update #-1:3e70e826:::osdmap.12:0# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9008) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3e70e826:::osdmap.12:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c000 allocated 0x[3000~19000] released 0x[] 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 freelist allocate 0x3000~19000 2019-12-07T19:33:38.530+0000 7fe2d959f700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 prepare 2019-12-07T19:33:38.530+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8106c000 2019-12-07T19:33:38.531+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 aio_wait 2019-12-07T19:33:38.531+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c000 2019-12-07T19:33:38.531+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 io_done 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:38.531+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 6836 < min_flush_size 524288 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125d180 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125d400 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125d680 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125d900 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125db80 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81796000 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81796280 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81796500 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81796780 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81796a00 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81796c80 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81796f00 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81797180 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81797400 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81797680 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81797900 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81797b80 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b8000 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b8280 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b8500 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b8780 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b8a00 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b8c80 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b8f00 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:33:38.531+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 6859 < min_flush_size 524288 2019-12-07T19:33:38.531+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 6859 < min_flush_size 524288 2019-12-07T19:33:38.531+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:38.531+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x22e2~1acb to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:38.531+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x22e2 0x22e2~1acb to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x22e2 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x2e2 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xdad and padding block with 0x253 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3dad 2019-12-07T19:33:38.531+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:38.531+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:38.531+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000173301s (0.000041267s flush + 0.000132034s kv commit) 2019-12-07T19:33:38.531+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:38.531+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:38.531+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8106c000> 2019-12-07T19:33:38.531+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:38.531+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 kv_submitted 2019-12-07T19:33:38.531+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c000 2019-12-07T19:33:38.531+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e8106c000 osr 0x558e8107a500 2019-12-07T19:33:38.531+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e8107b2c0) _discard drop 2 0x2000~1000 2019-12-07T19:33:38.531+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e8107b2c0) prepare_write seq 3 0x2000~1000 crc d9ad9fba 2019-12-07T19:33:38.531+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 0 _committed_osd_maps 1..12 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 0 advance to epoch 1 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 1 advance to epoch 2 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 2 advance to epoch 3 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 3 advance to epoch 4 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 4 advance to epoch 5 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 5 advance to epoch 6 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 6 advance to epoch 7 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 7 advance to epoch 8 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 8 advance to epoch 9 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 9 advance to epoch 10 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 10 advance to epoch 11 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 11 advance to epoch 12 (<= last 12 <= newest_map 12) 2019-12-07T19:33:38.531+0000 7fe2d7285700 0 osd.0 12 crush map has features 288514050185494528, adjusting msgr requires for clients 2019-12-07T19:33:38.531+0000 7fe2d7285700 0 osd.0 12 crush map has features 288514050185494528 was 288232575208792577, adjusting msgr requires for mons 2019-12-07T19:33:38.531+0000 7fe2d7285700 0 osd.0 12 crush map has features 3314932999778484224, adjusting msgr requires for osds 2019-12-07T19:33:38.531+0000 7fe2d7285700 1 osd.0 12 check_osdmap_features require_osd_release unknown -> octopus 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label got bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 7 meta) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 bluestore(/var/lib/ceph/osd/ceph-0/block) _write_bdev_label path /var/lib/ceph/osd/ceph-0/block label bdev(osd_uuid b39c74bd-b656-4393-a258-c2eebf51a555, size 0x1680000000, btime 2019-12-07T19:33:00.938659+0000, desc main, 8 meta) 2019-12-07T19:33:38.531+0000 7fe2d7285700 7 osd.0 12 consume_map version 12 2019-12-07T19:33:38.531+0000 7fe2d7285700 20 osd.0 12 prune_pg_created 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0:0.consume_map 12 (was 0) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0:1.consume_map 12 (was 0) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0:2.consume_map 12 (was 0) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0:3.consume_map 12 (was 0) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0:4.consume_map 12 (was 0) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0:5.consume_map 12 (was 0) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0:6.consume_map 12 (was 0) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0:7.consume_map 12 (was 0) 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 12 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:38.531+0000 7fe2d7285700 10 osd.0 12 _preboot _preboot mon has osdmaps 1..12 2019-12-07T19:33:38.531+0000 7fe2d7285700 5 osd.0 12 heartbeat osd_stat(store_statfs(0x163fffd000/0x40000000/0x1680000000, data 0x24e/0x1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [] op hist []) 2019-12-07T19:33:38.531+0000 7fe2d7285700 20 osd.0 12 check_full_status cur ratio 1.27157e-07, physical ratio 1.27157e-07, new state none 2019-12-07T19:33:38.531+0000 7fe2e149f700 10 osd.0 12 operator() waiting for peering work to drain 2019-12-07T19:33:38.531+0000 7fe2e149f700 10 osd.0 12 _send_boot 2019-12-07T19:33:38.531+0000 7fe2e149f700 20 osd.0 12 initial client_addrs [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079], cluster_addrs [v2:0.0.0.0:6822/37079,v1:0.0.0.0:6824/37079], hb_back_addrs [v2:0.0.0.0:6830/37079,v1:0.0.0.0:6832/37079], hb_front_addrs [v2:0.0.0.0:6826/37079,v1:0.0.0.0:6828/37079] 2019-12-07T19:33:38.531+0000 7fe2e149f700 1 -- [v2:0.0.0.0:6822/37079,v1:0.0.0.0:6824/37079] set_addr_unknowns [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] 2019-12-07T19:33:38.531+0000 7fe2e149f700 1 -- [v2:0.0.0.0:6822/37079,v1:0.0.0.0:6824/37079] set_addr_unknowns assuming my addr v2:0.0.0.0:6822/37079 matches provided addr v2:172.21.15.131:6818/37079 2019-12-07T19:33:38.531+0000 7fe2e149f700 1 -- [v2:0.0.0.0:6822/37079,v1:0.0.0.0:6824/37079] set_addr_unknowns assuming my addr v1:0.0.0.0:6824/37079 matches provided addr v2:172.21.15.131:6818/37079 2019-12-07T19:33:38.531+0000 7fe2e149f700 10 osd.0 12 new session (outgoing) 0x558e80245a00 con=0x558e802a5400 addr=v2:172.21.15.131:6822/37079 2019-12-07T19:33:38.531+0000 7fe2e149f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] set_addr_unknowns now [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] 2019-12-07T19:33:38.531+0000 7fe2e149f700 10 osd.0 12 assuming cluster_addrs match client_addrs [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] 2019-12-07T19:33:38.531+0000 7fe2e149f700 1 -- [v2:0.0.0.0:6830/37079,v1:0.0.0.0:6832/37079] set_addr_unknowns [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] 2019-12-07T19:33:38.531+0000 7fe2e149f700 1 -- [v2:0.0.0.0:6830/37079,v1:0.0.0.0:6832/37079] set_addr_unknowns assuming my addr v2:0.0.0.0:6830/37079 matches provided addr v2:172.21.15.131:6822/37079 2019-12-07T19:33:38.531+0000 7fe2e149f700 1 -- [v2:0.0.0.0:6830/37079,v1:0.0.0.0:6832/37079] set_addr_unknowns assuming my addr v1:0.0.0.0:6832/37079 matches provided addr v2:172.21.15.131:6822/37079 2019-12-07T19:33:38.531+0000 7fe2e149f700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] set_addr_unknowns now [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] 2019-12-07T19:33:38.531+0000 7fe2e149f700 10 osd.0 12 assuming hb_back_addrs match cluster_addrs [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] 2019-12-07T19:33:38.531+0000 7fe2e149f700 1 -- [v2:0.0.0.0:6826/37079,v1:0.0.0.0:6828/37079] set_addr_unknowns [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] 2019-12-07T19:33:38.531+0000 7fe2e149f700 1 -- [v2:0.0.0.0:6826/37079,v1:0.0.0.0:6828/37079] set_addr_unknowns assuming my addr v2:0.0.0.0:6826/37079 matches provided addr v2:172.21.15.131:6818/37079 2019-12-07T19:33:38.531+0000 7fe2e149f700 1 -- [v2:0.0.0.0:6826/37079,v1:0.0.0.0:6828/37079] set_addr_unknowns assuming my addr v1:0.0.0.0:6828/37079 matches provided addr v2:172.21.15.131:6818/37079 2019-12-07T19:33:38.531+0000 7fe2e149f700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] set_addr_unknowns now [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] 2019-12-07T19:33:38.531+0000 7fe2e149f700 10 osd.0 12 assuming hb_front_addrs match client_addrs [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] 2019-12-07T19:33:38.531+0000 7fe2e149f700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:38.532+0000 7fe2e149f700 1 osd.0 12 set_numa_affinity storage numa node 0 2019-12-07T19:33:38.532+0000 7fe2e149f700 1 osd.0 12 set_numa_affinity public network enp3s0f1 numa node 0 2019-12-07T19:33:38.532+0000 7fe2e149f700 1 osd.0 12 set_numa_affinity cluster network enp3s0f1 numa node 0 2019-12-07T19:33:38.532+0000 7fe2e149f700 1 osd.0 12 objectstore and network numa nodes all match 2019-12-07T19:33:38.532+0000 7fe2e149f700 1 osd.0 12 set_numa_affinity setting numa affinity to node 0 cpus 0-7 2019-12-07T19:33:38.532+0000 7fe2e149f700 10 osd.0 12 final client_addrs [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079], cluster_addrs [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079], hb_back_addrs [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079], hb_front_addrs [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] 2019-12-07T19:33:38.532+0000 7fe2e149f700 10 bluestore(/var/lib/ceph/osd/ceph-0) collect_metadata 2019-12-07T19:33:38.532+0000 7fe2e149f700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:38.532+0000 7fe2e149f700 1 bluestore(/var/lib/ceph/osd/ceph-0) collect_metadata devices span numa nodes 0 2019-12-07T19:33:38.532+0000 7fe2e149f700 10 bluestore(/var/lib/ceph/osd/ceph-0) collect_metadata 2019-12-07T19:33:38.532+0000 7fe2e149f700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_numa_node bdev nvme0n1 on numa_node 0 2019-12-07T19:33:38.532+0000 7fe2e149f700 1 bluestore(/var/lib/ceph/osd/ceph-0) collect_metadata devices span numa nodes 0 2019-12-07T19:33:38.533+0000 7fe2e149f700 10 osd.0 12 _collect_metadata {arch=x86_64,back_addr=[v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079],back_iface=enp3s0f1,bluefs=1,bluefs_dedicated_db=0,bluefs_dedicated_wal=0,bluefs_single_shared_device=1,bluestore_bdev_access_mode=file,bluestore_bdev_block_size=4096,bluestore_bdev_devices=nvme0n1,bluestore_bdev_driver=KernelDevice,bluestore_bdev_path=/var/lib/ceph/osd/ceph-0/block,bluestore_bdev_rotational=0,bluestore_bdev_size=96636764160,bluestore_bdev_support_discard=1,bluestore_bdev_type=ssd,ceph_release=octopus,ceph_version=ceph version 15.0.0-8038-g503c226 (503c226f594b239359b549e9328eeb7369e666df) octopus (dev),ceph_version_short=15.0.0-8038-g503c226,cpu=Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz,default_device_class=ssd,device_ids=nvme0n1=INTEL_SSDPEDMD400G4_CVFT623300N4400BGN,devices=nvme0n1,distro=rhel,distro_description=Employee SKU,distro_version=7.7,front_addr=[v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079],front_iface=enp3s0f1,hb_back_addr=[v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079],hb_front_addr=[v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079],hostname=smithi131,journal_rotational=0,kernel_description=#1 SMP Wed Sep 25 09:42:57 EDT 2019,kernel_version=3.10.0-1062.4.1.el7.x86_64,mem_swap_kb=0,mem_total_kb=32664044,network_numa_node=0,network_numa_nodes=0,numa_node=0,numa_node_cpus=0-7,objectstore_numa_node=0,objectstore_numa_nodes=0,os=Linux,osd_data=/var/lib/ceph/osd/ceph-0,osd_objectstore=bluestore,rotational=0} 2019-12-07T19:33:38.533+0000 7fe2e149f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- osd_boot(osd.0 booted 0 features 4540138292836696063 v12) v7 -- 0x558e81174700 con 0x558e80f39000 2019-12-07T19:33:38.549+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.583+0000 7fe2e2ca2700 10 osd.0 12 tick_without_osd_lock 2019-12-07T19:33:38.583+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffe4000/0x40000000/0x1680000000, data 0x544a/0x1a000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:38.599+0000 7fe2e4535700 10 osd.0 12 tick 2019-12-07T19:33:38.599+0000 7fe2e4535700 1 osd.0 12 tick checking mon for new map 2019-12-07T19:33:38.599+0000 7fe2e4535700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- mon_subscribe({osdmap=13}) v3 -- 0x558e8118e800 con 0x558e80f39000 2019-12-07T19:33:38.599+0000 7fe2e4535700 10 osd.0 12 do_waiters -- start 2019-12-07T19:33:38.599+0000 7fe2e4535700 10 osd.0 12 do_waiters -- finish 2019-12-07T19:33:38.599+0000 7fe2e4535700 20 osd.0 12 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:38.599+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.650+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.700+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.750+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.800+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.850+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.900+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:38.950+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.000+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.051+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.101+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.151+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.201+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.251+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.301+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.351+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.401+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.452+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.502+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415822 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 36480 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.519+0000 7fe2d5281700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.519+0000 7fe2d5281700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgropen(unknown.0) v3 -- 0x558e8027f8c0 con 0x558e81748000 2019-12-07T19:33:39.519+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.519+0000 7fe2d959f700 10 osd.0 12 ms_handle_connect con 0x558e81748000 2019-12-07T19:33:39.519+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 crc :-1 s=READY pgs=33 cs=0 l=1 rx=0 tx=0).ready entity=mgr.4099 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.519+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 msgr2=0x558e80efa680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 61 2019-12-07T19:33:39.519+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 msgr2=0x558e80efa680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:39.519+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 crc :-1 s=READY pgs=33 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:39.519+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 crc :-1 s=READY pgs=33 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:39.519+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 msgr2=0x558e80efa680 unknown :-1 s=STATE_CLOSED l=1).mark_down 2019-12-07T19:33:39.519+0000 7fe2d959f700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:39.522+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 10 ==== osd_map(13..13 src has 1..13) v4 ==== 3272+0+0 (crc 0 0 0) 0x558e817bb680 con 0x558e80f39000 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 osd.0 12 OSD::ms_dispatch: osd_map(13..13 src has 1..13) v4 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 osd.0 12 do_waiters -- start 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 osd.0 12 do_waiters -- finish 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 osd.0 12 _dispatch 0x558e817bb680 osd_map(13..13 src has 1..13) v4 2019-12-07T19:33:39.522+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x558e8118fc00 con 0x558e80f39000 2019-12-07T19:33:39.522+0000 7fe2d959f700 3 osd.0 12 handle_osd_map epochs [13,13], i have 12, src has [1,13] 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 osd.0 12 handle_osd_map got inc map for epoch 13 2019-12-07T19:33:39.522+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3e70e826:::osdmap.12:0# 0x0~0 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~8a8 size 0x8a8 (2216) 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81789c00 blob([0x1a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x8a8) SharedBlob(0x558e81789c70 sbid 0x0)) need 0x0~8a8 cache has 0x[0~8a8] 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3e70e826:::osdmap.12:0# 0x0~8a8 = 2216 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 osd.0 12 add_map_bl 12 2216 bytes 2019-12-07T19:33:39.522+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e802a4c00 0x558e80efb700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 osd.0 12 got_full_map 13, nothing requested 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 osd.0 12 handle_osd_map pg_num_history pg_num_history(e13 pg_nums {} deleted_pools ) 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps purged_snaps {13={}} 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 osd.0 12 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e13 [1,13] lci=[0,0]) 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e817de000 seq 6 2019-12-07T19:33:39.522+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e802a4c00 0x558e80efb700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:cc63972f:::inc_osdmap.13:0# key 0x7f7fffffffffffffffcc6397'/!inc_osdmap.13!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:39.522+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:cc63972f:::inc_osdmap.13:0# 0x0~c9c 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1053 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:cc63972f:::inc_osdmap.13:0# 0x0~c9c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~c9c 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0x364 on front/back, now 0x0~1000 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e817de000 1 blobs 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x1c000~1000/1000,1000,0 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x1c000~1000] 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e811d0fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d1180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e811d0fc0 blob([0x1c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d1180 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore.blob(0x558e811d0fc0) get_ref 0x0~c9c Blob(0x558e811d0fc0 blob([0x1c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d1180 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore.blob(0x558e811d0fc0) get_ref init 0x1000, 1000 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~c9c: 0x0~c9c Blob(0x558e811d0fc0 blob([0x1c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0xc9c) SharedBlob(0x558e811d1180 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e811d1198 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0xc9c 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9508) dirty_range mark inline shard dirty 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:cc63972f:::inc_osdmap.13:0# 0x0~c9c = 0 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:30f0e826:::osdmap.13:0# key 0x7f7fffffffffffffff30f0e8'&!osdmap.13!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:39.522+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:30f0e826:::osdmap.13:0# 0x0~1168 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1054 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:30f0e826:::osdmap.13:0# 0x0~1168 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~168 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xe98 on front/back, now 0x0~1000 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e817de000 2 blobs 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x1d000~2000/1000,2000,0 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x1d000~2000] 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e811d1500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d16c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e811d1500 blob([0x1d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d16c0 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore.blob(0x558e811d1500) get_ref 0x0~1000 Blob(0x558e811d1500 blob([0x1d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d16c0 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore.blob(0x558e811d1500) get_ref init 0x1000, 1000 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e811d1500 blob([0x1d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e811d16c0 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e811d16d8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e811d1810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d1a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e811d1810 blob([!~1000,0x1e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d1a40 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore.blob(0x558e811d1810) get_ref 0x1000~168 Blob(0x558e811d1810 blob([!~1000,0x1e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e811d1a40 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore.blob(0x558e811d1810) get_ref init 0x2000, 1000 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~168: 0x1000~168 Blob(0x558e811d1810 blob([!~1000,0x1e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x558e811d1a40 sbid 0x0)) 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e811d1a58 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1168 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9788) dirty_range mark inline shard dirty 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:30f0e826:::osdmap.13:0# 0x0~1168 = 0 2019-12-07T19:33:39.522+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:39.522+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:39.522+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~12 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfee 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~12: 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e817de000 0 blobs 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:33:39.523+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:39.523+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:39.523+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:39.523+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:39.523+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:39.523+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:39.523+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e817de000 0 blobs 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:33:39.523+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:39.523+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e817de000 cost 25145 (4 ios * 4000 + 9145 bytes) 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e817de000 onodes 0x558e80251b80,0x558e817b9180,0x558e817b9400,0x558e817b9680 shared_blobs 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9508) update #-1:cc63972f:::inc_osdmap.13:0# 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9508) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:cc63972f:::inc_osdmap.13:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9788) update #-1:30f0e826:::osdmap.13:0# 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9788) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:30f0e826:::osdmap.13:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e817de000 allocated 0x[1c000~3000] released 0x[] 2019-12-07T19:33:39.523+0000 7fe2d959f700 10 freelist allocate 0x1c000~3000 2019-12-07T19:33:39.523+0000 7fe2d959f700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:39.523+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de000 prepare 2019-12-07T19:33:39.523+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e817de000 2019-12-07T19:33:39.523+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e81748c00 0x558e817ee000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.523+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de000 aio_wait 2019-12-07T19:33:39.523+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e817de000 2019-12-07T19:33:39.523+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de000 io_done 2019-12-07T19:33:39.523+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:39.523+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:39.523+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:39.523+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e81749000 0x558e817ee580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.523+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8788 < min_flush_size 524288 2019-12-07T19:33:39.523+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:33:39.523+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:33:39.523+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9400 had 1 2019-12-07T19:33:39.523+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9680 had 1 2019-12-07T19:33:39.523+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:33:39.523+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:39.523+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:39.523+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8811 < min_flush_size 524288 2019-12-07T19:33:39.523+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8811 < min_flush_size 524288 2019-12-07T19:33:39.523+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:39.523+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3dad~226b to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:39.523+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3dad 0x3dad~226b to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:39.524+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:39.524+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x3dad 2019-12-07T19:33:39.524+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xdad 2019-12-07T19:33:39.524+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:39.524+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x18 and padding block with 0xfe8 2019-12-07T19:33:39.524+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x6018 2019-12-07T19:33:39.524+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:39.524+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:39.524+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:39.524+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:39.524+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000309580s (0.000060761s flush + 0.000248819s kv commit) 2019-12-07T19:33:39.524+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:39.524+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:39.524+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e817de000> 2019-12-07T19:33:39.524+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:39.524+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de000 kv_submitted 2019-12-07T19:33:39.524+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e817de000 2019-12-07T19:33:39.524+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e817de000 osr 0x558e8107a500 2019-12-07T19:33:39.524+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e8107b2c0) prepare_write seq 4 0x1b000~1000 crc 2ba63da0 2019-12-07T19:33:39.524+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e8107b2c0) _discard drop 3 0x2000~1000 2019-12-07T19:33:39.524+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e8107b2c0) prepare_write seq 4 0x2000~1000 crc 75bc8a9b 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 12 _committed_osd_maps 13..13 2019-12-07T19:33:39.524+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 12 advance to epoch 13 (<= last 13 <= newest_map 13) 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 up_epoch is 13 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 boot_epoch is 13 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 osd.0 13 state: booting -> active 2019-12-07T19:33:39.524+0000 7fe2d7285700 7 osd.0 13 consume_map version 13 2019-12-07T19:33:39.524+0000 7fe2d7285700 20 osd.0 13 prune_pg_created 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0:0.consume_map 13 (was 12) 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0:1.consume_map 13 (was 12) 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0:2.consume_map 13 (was 12) 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0:3.consume_map 13 (was 12) 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0:4.consume_map 13 (was 12) 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0:5.consume_map 13 (was 12) 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0:6.consume_map 13 (was 12) 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0:7.consume_map 13 (was 12) 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 maybe_update_heartbeat_peers updating 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 adding neighbor peer osd.1 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 --2- >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558e81749800 0x558e817eeb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 --2- >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e81749c00 0x558e817ef080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 _add_heartbeat_peer: new peer osd.1 v2:172.21.15.131:6831/37078 v2:172.21.15.131:6827/37078 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 adding neighbor peer osd.5 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 --2- >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558e817fe000 0x558e817ef600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 --2- >> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] conn(0x558e817fe400 0x558e817efb80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 _add_heartbeat_peer: new peer osd.5 v2:172.21.15.109:6806/36936 v2:172.21.15.109:6804/36936 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 adding neighbor peer osd.7 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 --2- >> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] conn(0x558e817fe800 0x558e817f0100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 --2- >> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] conn(0x558e817fec00 0x558e817f0680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 _add_heartbeat_peer: new peer osd.7 v2:172.21.15.109:6830/36939 v2:172.21.15.109:6828/36939 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 adding random peer osd.2 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 --2- >> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] conn(0x558e817ff000 0x558e817f0c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 --2- >> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] conn(0x558e817ff400 0x558e817f1180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 _add_heartbeat_peer: new peer osd.2 v2:172.21.15.131:6808/37077 v2:172.21.15.131:6806/37077 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 adding random peer osd.3 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 --2- >> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] conn(0x558e817ff800 0x558e817f1700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 --2- >> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] conn(0x558e817ffc00 0x558e81816000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 _add_heartbeat_peer: new peer osd.3 v2:172.21.15.131:6816/37076 v2:172.21.15.131:6814/37076 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 adding random peer osd.4 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 --2- >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x558e8181a000 0x558e81816580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 --2- >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x558e8181a400 0x558e81816b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 _add_heartbeat_peer: new peer osd.4 v2:172.21.15.109:6814/36938 v2:172.21.15.109:6812/36938 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 adding random peer osd.6 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 --2- >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558e8181a800 0x558e81817080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7fe2d7285700 1 --2- >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558e8181ac00 0x558e81817600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 _add_heartbeat_peer: new peer osd.6 v2:172.21.15.109:6822/36937 v2:172.21.15.109:6820/36937 2019-12-07T19:33:39.524+0000 7fe2d7285700 10 osd.0 13 maybe_update_heartbeat_peers 7 peers, extras 1,2,3,4,5,6,7 2019-12-07T19:33:39.524+0000 7fe2d7285700 7 osd.0 13 activate_map version 13 2019-12-07T19:33:39.524+0000 7fe2cc26f700 20 osd.0 13 agent_entry empty queue 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e81749400 0x558e81817b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e8181b800 0x558e81818100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e8181bc00 0x558e81818680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e8182c000 0x558e81818c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e8182c400 0x558e81819180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e81749400 0x558e81817b80 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e8181b800 0x558e81818100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e8182c400 0x558e81819180 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e8182c800 0x558e81819700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e8182cc00 0x558e81844000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.535+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 11 ==== osd_map(13..13 src has 1..13) v4 ==== 4500+0+0 (crc 0 0 0) 0x558e817bb8c0 con 0x558e80f39000 2019-12-07T19:33:39.535+0000 7fe2d959f700 20 osd.0 13 OSD::ms_dispatch: osd_map(13..13 src has 1..13) v4 2019-12-07T19:33:39.535+0000 7fe2d959f700 10 osd.0 13 do_waiters -- start 2019-12-07T19:33:39.535+0000 7fe2d959f700 10 osd.0 13 do_waiters -- finish 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e8182d000 0x558e81844580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.535+0000 7fe2d959f700 20 osd.0 13 _dispatch 0x558e817bb8c0 osd_map(13..13 src has 1..13) v4 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e8182d400 0x558e81844b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.535+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e81748c00 0x558e817ee000 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e8182d800 0x558e81845080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.535+0000 7fe2d959f700 3 osd.0 13 handle_osd_map epochs [13,13], i have 13, src has [1,13] 2019-12-07T19:33:39.535+0000 7fe2d959f700 10 osd.0 13 no new maps here, dropping 2019-12-07T19:33:39.535+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e81749000 0x558e817ee580 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e8182dc00 0x558e81845600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558e817fe000 0x558e817ef600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7fe2e6d3a700 1 --2- >> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] conn(0x558e817fe800 0x558e817f0100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7fe2e6d3a700 1 --2- >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x558e8181a000 0x558e81816580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e8182c000 0x558e81818c00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- >> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] conn(0x558e817fec00 0x558e817f0680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e8182d000 0x558e81844580 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7fe2e7d3c700 1 --2- >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x558e8181a400 0x558e81816b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.535+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e8182dc00 0x558e81845600 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7fe2e7d3c700 1 --2- >> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] conn(0x558e817ff800 0x558e817f1700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 1 --2- >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558e8181ac00 0x558e81817600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7fe2e7d3c700 1 --2- >> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] conn(0x558e817ff800 0x558e817f1700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).handle_hello peer v2:172.21.15.131:6816/37076 says I am v2:172.21.15.131:51682/0 (socket says 172.21.15.131:51682) 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication new session 0x558e81807380 con 0x558e81748c00 entity osd.7 addr 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication session 0x558e81807380 osd.7 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication new session 0x558e81807520 con 0x558e81749000 entity osd.7 addr 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication session 0x558e81807520 osd.7 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 1 --2- >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558e8181ac00 0x558e81817600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).handle_hello peer v2:172.21.15.109:6820/36937 says I am v2:172.21.15.131:52360/0 (socket says 172.21.15.131:52360) 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 learned_addr learned my addr 172.21.15.131:0/37079 (peer_addr_for_me v2:172.21.15.131:0/0) 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication new session 0x558e818076c0 con 0x558e8182d000 entity osd.4 addr 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication session 0x558e818076c0 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] conn(0x558e817ff400 0x558e817f1180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7fe2e753b700 10 osd.0 13 ms_handle_authentication new session 0x558e81807860 con 0x558e802a4c00 entity osd.6 addr 2019-12-07T19:33:39.536+0000 7fe2e753b700 10 osd.0 13 ms_handle_authentication session 0x558e81807860 osd.6 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.109:0/36939 conn(0x558e81748c00 0x558e817ee000 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication new session 0x558e81807a00 con 0x558e8182c000 entity osd.2 addr 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication session 0x558e81807a00 osd.2 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.536+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] conn(0x558e817fe400 0x558e817efb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e8181bc00 0x558e81818680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.109:0/36939 conn(0x558e81749000 0x558e817ee580 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication new session 0x558e81807ba0 con 0x558e8182dc00 entity osd.1 addr 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication session 0x558e81807ba0 osd.1 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.536+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e8182cc00 0x558e81844000 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e81749c00 0x558e817ef080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e8182d800 0x558e81845080 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.109:0/36938 conn(0x558e8182d000 0x558e81844580 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.536+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558e8181a800 0x558e81817080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] conn(0x558e817ff000 0x558e817f0c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.536+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] conn(0x558e817fe800 0x558e817f0100 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.536+0000 7fe2e753b700 10 osd.0 13 ms_handle_authentication new session 0x558e818c2000 con 0x558e8181bc00 entity osd.4 addr 2019-12-07T19:33:39.536+0000 7fe2e753b700 10 osd.0 13 ms_handle_authentication session 0x558e818c2000 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.537+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.131:0/37078 conn(0x558e8182dc00 0x558e81845600 crc :-1 s=READY pgs=4 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558e81749800 0x558e817eeb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.537+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.131:0/37077 conn(0x558e8182c000 0x558e81818c00 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558e8181ac00 0x558e81817600 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.109:0/36938 conn(0x558e8181bc00 0x558e81818680 crc :-1 s=READY pgs=4 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7fe2e6d3a700 0 -- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e81749c00 msgr2=0x558e817ef080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2019-12-07T19:33:39.537+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e81749c00 msgr2=0x558e817ef080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send send error: (32) Broken pipe 2019-12-07T19:33:39.537+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e81749c00 0x558e817ef080 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).write client ident write failed r=-32 ((32) Broken pipe) 2019-12-07T19:33:39.537+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.109:0/36937 conn(0x558e802a4c00 0x558e80efb700 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e81749c00 0x558e817ef080 crc :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.200000 2019-12-07T19:33:39.537+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] conn(0x558e8181a000 0x558e81816580 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7fe2e753b700 10 osd.0 13 ms_handle_authentication new session 0x558e818c21a0 con 0x558e8182cc00 entity osd.6 addr 2019-12-07T19:33:39.537+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] conn(0x558e817ff400 0x558e817f1180 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7fe2e753b700 10 osd.0 13 ms_handle_authentication session 0x558e818c21a0 osd.6 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.537+0000 7fe2e753b700 10 osd.0 13 ms_handle_authentication new session 0x558e818c2340 con 0x558e8182d800 entity osd.3 addr 2019-12-07T19:33:39.537+0000 7fe2e753b700 10 osd.0 13 ms_handle_authentication session 0x558e818c2340 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.537+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.109:0/36937 conn(0x558e8182cc00 0x558e81844000 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.131:0/37076 conn(0x558e8182d800 0x558e81845080 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] conn(0x558e817ffc00 0x558e81816000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.537+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] conn(0x558e817fe400 0x558e817efb80 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558e81749800 0x558e817eeb00 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558e8181a800 0x558e81817080 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.536+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 learned_addr learned my addr 172.21.15.131:0/37079 (peer_addr_for_me v2:172.21.15.131:0/0) 2019-12-07T19:33:39.537+0000 7fe2e7d3c700 10 osd.0 13 ms_handle_authentication new session 0x558e818c24e0 con 0x558e8182c400 entity osd.3 addr 2019-12-07T19:33:39.537+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] conn(0x558e817ff000 0x558e817f0c00 crc :-1 s=READY pgs=8 cs=0 l=1 rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7fe2e7d3c700 10 osd.0 13 ms_handle_authentication session 0x558e818c24e0 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.537+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e8182c800 0x558e81819700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.537+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e8182d400 0x558e81844b00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.537+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.131:0/37076 conn(0x558e8182c400 0x558e81819180 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.537+0000 7fe2e7d3c700 10 osd.0 13 ms_handle_authentication new session 0x558e818c2680 con 0x558e8181b800 entity osd.5 addr 2019-12-07T19:33:39.537+0000 7fe2e7d3c700 10 osd.0 13 ms_handle_authentication session 0x558e818c2680 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.537+0000 7fe2e7d3c700 10 osd.0 13 ms_handle_authentication new session 0x558e818c2820 con 0x558e81749400 entity osd.1 addr 2019-12-07T19:33:39.537+0000 7fe2e7d3c700 10 osd.0 13 ms_handle_authentication session 0x558e818c2820 osd.1 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.538+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] conn(0x558e817ffc00 0x558e81816000 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.538+0000 7fe2e7d3c700 10 osd.0 13 ms_handle_authentication new session 0x558e818c29c0 con 0x558e8182d400 entity osd.2 addr 2019-12-07T19:33:39.538+0000 7fe2e7d3c700 10 osd.0 13 ms_handle_authentication session 0x558e818c29c0 osd.2 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.538+0000 7fe2e7d3c700 10 osd.0 13 ms_handle_authentication new session 0x558e818c2b60 con 0x558e8182c800 entity osd.5 addr 2019-12-07T19:33:39.538+0000 7fe2e7d3c700 10 osd.0 13 ms_handle_authentication session 0x558e818c2b60 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:39.538+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.109:0/36936 conn(0x558e8181b800 0x558e81818100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.538+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.131:0/37078 conn(0x558e81749400 0x558e81817b80 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.538+0000 7fe2e7d3c700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] conn(0x558e817ff800 0x558e817f1700 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.538+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.131:0/37077 conn(0x558e8182d400 0x558e81844b00 crc :-1 s=READY pgs=4 cs=0 l=1 rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.538+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.109:0/36936 conn(0x558e8182c800 0x558e81819700 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.538+0000 7fe2e7d3c700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] conn(0x558e8181a400 0x558e81816b00 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.538+0000 7fe2e7d3c700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] conn(0x558e817fec00 0x558e817f0680 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.538+0000 7fe2e7d3c700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] conn(0x558e817fe000 0x558e817ef600 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.552+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 1 osrs, 4 txcs 2019-12-07T19:33:39.552+0000 7fe2dc28f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock osr 0x558e8107a500 2 ios pending 2019-12-07T19:33:39.552+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 4 0x2000~1000 2019-12-07T19:33:39.552+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x2000~1000 crc 75bc8a9b 2019-12-07T19:33:39.552+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 4 0x1b000~1000 2019-12-07T19:33:39.552+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x1b000~1000 crc 2ba63da0 2019-12-07T19:33:39.552+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.552+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish osr 0x558e8107a500 2019-12-07T19:33:39.552+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish dequeueing 2019-12-07T19:33:39.556+0000 7fe2e4535700 10 osd.0 13 tick 2019-12-07T19:33:39.556+0000 7fe2e4535700 10 osd.0 13 do_waiters -- start 2019-12-07T19:33:39.556+0000 7fe2e4535700 10 osd.0 13 do_waiters -- finish 2019-12-07T19:33:39.556+0000 7fe2e4535700 20 osd.0 13 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 10 osd.0 13 tick_without_osd_lock 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffe1000/0x40000000/0x1680000000, data 0x724e/0x1d000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 25 osd.0 13 heartbeat_check we haven't sent ping to osd.1 yet, skipping 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 25 osd.0 13 heartbeat_check we haven't sent ping to osd.2 yet, skipping 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 25 osd.0 13 heartbeat_check we haven't sent ping to osd.3 yet, skipping 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 25 osd.0 13 heartbeat_check we haven't sent ping to osd.4 yet, skipping 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 25 osd.0 13 heartbeat_check we haven't sent ping to osd.5 yet, skipping 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 25 osd.0 13 heartbeat_check we haven't sent ping to osd.6 yet, skipping 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 25 osd.0 13 heartbeat_check we haven't sent ping to osd.7 yet, skipping 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 20 osd.0 13 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 20 osd.0 13 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 20 osd.0 13 scrub_load_below_threshold loadavg per cpu 0.06875 < max 5 = yes 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 20 osd.0 13 sched_scrub load_is_low=1 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 20 osd.0 13 sched_scrub done 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 10 osd.0 13 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 20 osd.0 13 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 10 osd.0 13 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 20 osd.0 13 send_beacon sending 2019-12-07T19:33:39.567+0000 7fe2e2ca2700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- osd_beacon(pgs [] lec 0 last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 v13) v2 -- 0x558e8118e800 con 0x558e80f39000 2019-12-07T19:33:39.602+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.652+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.702+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.738+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e81749c00 0x558e817ef080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:39.738+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e81749c00 0x558e817ef080 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:39.752+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.802+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.853+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.903+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:39.953+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.003+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.053+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.103+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.153+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.203+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.254+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.304+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.354+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.404+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.454+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.504+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.519+0000 7fe2d5281700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:40.519+0000 7fe2d5281700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgropen(unknown.0) v3 -- 0x558e8027f600 con 0x558e81748000 2019-12-07T19:33:40.519+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:40.519+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] conn(0x558e81748000 0x558e80efa680 crc :-1 s=READY pgs=40 cs=0 l=1 rx=0 tx=0).ready entity=mgr.4099 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:40.519+0000 7fe2d959f700 10 osd.0 13 ms_handle_connect con 0x558e81748000 2019-12-07T19:33:40.520+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mgr.4099 v2:172.21.15.131:6800/36137 1 ==== mgrconfigure(period=5, threshold=5) v3 ==== 12+0+0 (crc 0 0 0) 0x558e80264a00 con 0x558e81748000 2019-12-07T19:33:40.520+0000 7fe2d959f700 10 osd.0 13 setting 0 queries 2019-12-07T19:33:40.520+0000 7fe2d959f700 20 osd.0 13 reports for 0 queries 2019-12-07T19:33:40.520+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.0 +109-0 packed 1198 daemon_metrics=2) v8 -- 0x558e81174380 con 0x558e81748000 2019-12-07T19:33:40.520+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x558e81174700 con 0x558e81748000 2019-12-07T19:33:40.554+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.579+0000 7fe2e2ca2700 10 osd.0 13 tick_without_osd_lock 2019-12-07T19:33:40.579+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffe1000/0x40000000/0x1680000000, data 0x724e/0x1d000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:40.579+0000 7fe2e2ca2700 25 osd.0 13 heartbeat_check we haven't sent ping to osd.1 yet, skipping 2019-12-07T19:33:40.579+0000 7fe2e2ca2700 25 osd.0 13 heartbeat_check we haven't sent ping to osd.2 yet, skipping 2019-12-07T19:33:40.579+0000 7fe2e2ca2700 25 osd.0 13 heartbeat_check we haven't sent ping to osd.3 yet, skipping 2019-12-07T19:33:40.579+0000 7fe2e2ca2700 25 osd.0 13 heartbeat_check we haven't sent ping to osd.4 yet, skipping 2019-12-07T19:33:40.579+0000 7fe2e2ca2700 25 osd.0 13 heartbeat_check we haven't sent ping to osd.5 yet, skipping 2019-12-07T19:33:40.579+0000 7fe2e2ca2700 25 osd.0 13 heartbeat_check we haven't sent ping to osd.6 yet, skipping 2019-12-07T19:33:40.579+0000 7fe2e2ca2700 25 osd.0 13 heartbeat_check we haven't sent ping to osd.7 yet, skipping 2019-12-07T19:33:40.579+0000 7fe2e2ca2700 20 osd.0 13 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:40.580+0000 7fe2e2ca2700 20 osd.0 13 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:40.580+0000 7fe2e2ca2700 20 osd.0 13 scrub_load_below_threshold loadavg per cpu 0.06875 < max 5 = yes 2019-12-07T19:33:40.580+0000 7fe2e2ca2700 20 osd.0 13 sched_scrub load_is_low=1 2019-12-07T19:33:40.580+0000 7fe2e2ca2700 20 osd.0 13 sched_scrub done 2019-12-07T19:33:40.580+0000 7fe2e2ca2700 10 osd.0 13 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:40.580+0000 7fe2e2ca2700 20 osd.0 13 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:40.580+0000 7fe2e2ca2700 10 osd.0 13 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:40.589+0000 7fe2e4535700 10 osd.0 13 tick 2019-12-07T19:33:40.589+0000 7fe2e4535700 10 osd.0 13 do_waiters -- start 2019-12-07T19:33:40.589+0000 7fe2e4535700 10 osd.0 13 do_waiters -- finish 2019-12-07T19:33:40.589+0000 7fe2e4535700 20 osd.0 13 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:40.604+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.655+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.705+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.755+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.805+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.855+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.905+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:40.955+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.005+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.056+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.106+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.156+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.206+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 39716 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.235+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.2 172.21.15.131:0/37077 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.235503+0000/4.707221878s send_stamp 4.707221878s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558e818a8600 con 0x558e8182c000 2019-12-07T19:33:41.235+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.2 172.21.15.131:0/37077 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.235503+0000/4.707221878s send_stamp 4.707221878s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558e8118ea00 con 0x558e8182d400 2019-12-07T19:33:41.235+0000 7fe2e6d3a700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016685233s,]) 2019-12-07T19:33:41.235+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.235503+0000/4.707221878s send_stamp 4.690536645s delta_ub 18446744073.692866383s) v5 -- 0x558e818a8c00 con 0x558e8182c000 2019-12-07T19:33:41.235+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558e818d6000 0x558e81845b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.235+0000 7fe2e7d3c700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016624923s,]) 2019-12-07T19:33:41.235+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.235503+0000/4.707221878s send_stamp 4.690596955s delta_ub 18446744073.692926693s) v5 -- 0x558e8118e800 con 0x558e8182d400 2019-12-07T19:33:41.235+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558e818d6000 0x558e81845b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.235+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e818d6400 0x558e81846100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.235+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e818d6400 0x558e81846100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.236+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558e818d6000 0x558e81845b80 crc :-1 s=START_CONNECT pgs=0 cs=0 l=0 rx=0 tx=0)._fault waiting 15.000000 2019-12-07T19:33:41.236+0000 7fe2e753b700 10 osd.0 13 ms_handle_authentication new session 0x558e81807d40 con 0x558e818d6400 entity osd.2 addr 2019-12-07T19:33:41.236+0000 7fe2e753b700 10 osd.0 13 ms_handle_authentication session 0x558e81807d40 osd.2 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.236+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558e818d6400 0x558e81846100 crc :-1 s=SESSION_ACCEPTING pgs=3 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, replacing existing=0x558e818d6000 socket by this connection's socket 2019-12-07T19:33:41.236+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558e818d6400 0x558e81846100 crc :-1 s=SESSION_ACCEPTING pgs=3 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.236+0000 7fe2d4a80700 2 osd.0 13 ms_handle_reset con 0x558e818d6400 session 0x558e81807d40 2019-12-07T19:33:41.236+0000 7fe2e753b700 10 osd.0 13 new session (incoming)0x558e818c2d00 con=0x558e818d6000 addr=v2:172.21.15.131:6804/37077 must have raced with connect 2019-12-07T19:33:41.236+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] conn(0x558e818d6000 0x558e81845b80 unknown :-1 s=READY pgs=3 cs=0 l=0 rx=0 tx=0).ready entity=osd.2 client_cookie=8f213ff6e35a40c2 server_cookie=f8d0c8aac30d53ca in_seq=0 out_seq=0 2019-12-07T19:33:41.240+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.5 172.21.15.109:0/36936 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.240546+0000/4.706869389s send_stamp 4.706869389s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558e8118fc00 con 0x558e8181b800 2019-12-07T19:33:41.240+0000 7fe2e7d3c700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011383076s,]) 2019-12-07T19:33:41.240+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.240546+0000/4.706869389s send_stamp 4.695486313s delta_ub 18446744073.698168540s) v5 -- 0x558e8118fa00 con 0x558e8181b800 2019-12-07T19:33:41.240+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558e818d6800 0x558e81846680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.240+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.5 172.21.15.109:0/36936 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.240546+0000/4.706869389s send_stamp 4.706869389s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558e8118fc00 con 0x558e8182c800 2019-12-07T19:33:41.240+0000 7fe2e7d3c700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011320768s,]) 2019-12-07T19:33:41.240+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.240546+0000/4.706869389s send_stamp 4.695548621s delta_ub 18446744073.698230848s) v5 -- 0x558e8118e800 con 0x558e8182c800 2019-12-07T19:33:41.240+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558e818d6800 0x558e81846680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.240+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e818d6c00 0x558e81846c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.241+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558e818d6800 0x558e81846680 crc :-1 s=START_CONNECT pgs=0 cs=0 l=0 rx=0 tx=0)._fault waiting 15.000000 2019-12-07T19:33:41.241+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e818d6c00 0x558e81846c00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.241+0000 7fe2e753b700 10 osd.0 13 ms_handle_authentication new session 0x558e818c2ea0 con 0x558e818d6c00 entity osd.5 addr 2019-12-07T19:33:41.241+0000 7fe2e753b700 10 osd.0 13 ms_handle_authentication session 0x558e818c2ea0 osd.5 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.241+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558e818d6c00 0x558e81846c00 crc :-1 s=SESSION_ACCEPTING pgs=6 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, replacing existing=0x558e818d6800 socket by this connection's socket 2019-12-07T19:33:41.241+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558e818d6c00 0x558e81846c00 crc :-1 s=SESSION_ACCEPTING pgs=6 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.241+0000 7fe2d4a80700 2 osd.0 13 ms_handle_reset con 0x558e818d6c00 session 0x558e818c2ea0 2019-12-07T19:33:41.241+0000 7fe2e753b700 10 osd.0 13 new session (incoming)0x558e818c3040 con=0x558e818d6800 addr=v2:172.21.15.109:6802/36936 must have raced with connect 2019-12-07T19:33:41.241+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] conn(0x558e818d6800 0x558e81846680 unknown :-1 s=READY pgs=6 cs=0 l=0 rx=0 tx=0).ready entity=osd.5 client_cookie=faf79c1ac078c45f server_cookie=e9d4245ca5f19a39 in_seq=0 out_seq=0 2019-12-07T19:33:41.249+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.1 172.21.15.131:0/37078 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.706948734s send_stamp 4.706948734s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558e818a8a00 con 0x558e8182dc00 2019-12-07T19:33:41.249+0000 7fe2e6d3a700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002015933s,]) 2019-12-07T19:33:41.249+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.706948734s send_stamp 4.704932801s delta_ub 18446744073.707535683s) v5 -- 0x558e818a8c00 con 0x558e8182dc00 2019-12-07T19:33:41.249+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558e818d7c00 0x558e818e4000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.249+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.1 172.21.15.131:0/37078 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.706948734s send_stamp 4.706948734s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558e8118fc00 con 0x558e81749400 2019-12-07T19:33:41.249+0000 7fe2e7d3c700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.001961295s,]) 2019-12-07T19:33:41.249+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.706948734s send_stamp 4.704987439s delta_ub 18446744073.707590321s) v5 -- 0x558e818a9200 con 0x558e81749400 2019-12-07T19:33:41.249+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.3 172.21.15.131:0/37076 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.707537742s send_stamp 4.707537742s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558e8118fc00 con 0x558e8182c400 2019-12-07T19:33:41.249+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.3 172.21.15.131:0/37076 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.707537742s send_stamp 4.707537742s) v5 ==== 2025+0+0 (crc 0 0 0) 0x558e818a9600 con 0x558e8182d800 2019-12-07T19:33:41.249+0000 7fe2e7d3c700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002500751s,]) 2019-12-07T19:33:41.249+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.707537742s send_stamp 4.705036991s delta_ub 18446744073.707050865s) v5 -- 0x558e8118e800 con 0x558e8182c400 2019-12-07T19:33:41.249+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.249+0000 7fe2e753b700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002453822s,]) 2019-12-07T19:33:41.249+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.250025+0000/4.707537742s send_stamp 4.705083920s delta_ub 18446744073.707097794s) v5 -- 0x558e818a9800 con 0x558e8182d800 2019-12-07T19:33:41.250+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.250+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e818d7400 0x558e81847700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.250+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558e818d7c00 0x558e818e4000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.250+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e818d7400 0x558e81847700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.250+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e818d7800 0x558e818e4b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.250+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e818d7800 0x558e818e4b00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.250+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 crc :-1 s=START_CONNECT pgs=0 cs=0 l=0 rx=0 tx=0)._fault waiting 15.000000 2019-12-07T19:33:41.250+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication new session 0x558e818c2ea0 con 0x558e818d7400 entity osd.1 addr 2019-12-07T19:33:41.250+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication session 0x558e818c2ea0 osd.1 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.250+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication new session 0x558e818c3380 con 0x558e818d7800 entity osd.3 addr 2019-12-07T19:33:41.250+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication session 0x558e818c3380 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.251+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558e818d7400 0x558e81847700 crc :-1 s=SESSION_ACCEPTING pgs=6 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, replacing existing=0x558e818d7c00 socket by this connection's socket 2019-12-07T19:33:41.251+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558e818d7400 0x558e81847700 crc :-1 s=SESSION_ACCEPTING pgs=6 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.251+0000 7fe2d4a80700 2 osd.0 13 ms_handle_reset con 0x558e818d7400 session 0x558e818c2ea0 2019-12-07T19:33:41.251+0000 7fe2e6d3a700 10 osd.0 13 new session (incoming)0x558e818c3520 con=0x558e818d7c00 addr=v2:172.21.15.131:6823/37078 must have raced with connect 2019-12-07T19:33:41.251+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558e818d7c00 0x558e818e4000 unknown :-1 s=READY pgs=6 cs=0 l=0 rx=0 tx=0).ready entity=osd.1 client_cookie=1233dd104d1dac98 server_cookie=89d5462ffa9332cd in_seq=0 out_seq=0 2019-12-07T19:33:41.251+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7800 0x558e818e4b00 crc :-1 s=SESSION_ACCEPTING pgs=7 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, replacing existing=0x558e818d7000 socket by this connection's socket 2019-12-07T19:33:41.251+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7800 0x558e818e4b00 crc :-1 s=SESSION_ACCEPTING pgs=7 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.251+0000 7fe2d4a80700 2 osd.0 13 ms_handle_reset con 0x558e818d7800 session 0x558e818c3380 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 5 osd.0 13 heartbeat osd_stat(store_statfs(0x163ffe1000/0x40000000/0x1680000000, data 0x724e/0x1d000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5,6,7] op hist []) 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 20 osd.0 13 check_full_status cur ratio 1.31395e-06, physical ratio 1.31395e-06, new state none 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s delta_ub 18446744073.707590321s) v5 -- 0x558e818a9a00 con 0x558e81749800 2019-12-07T19:33:41.251+0000 7fe2e6d3a700 10 osd.0 13 new session (incoming)0x558e81807d40 con=0x558e818d7000 addr=v2:172.21.15.131:6812/37076 must have raced with connect 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s delta_ub 18446744073.707590321s) v5 -- 0x558e818a9c00 con 0x558e81749c00 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s delta_ub 18446744073.692926693s) v5 -- 0x558e818a9e00 con 0x558e817ff000 2019-12-07T19:33:41.251+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=READY pgs=7 cs=0 l=0 rx=0 tx=0).ready entity=osd.3 client_cookie=f8db806cc22eb7e9 server_cookie=47e95957322ca882 in_seq=0 out_seq=0 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s delta_ub 18446744073.692926693s) v5 -- 0x558e818ee000 con 0x558e817ff400 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s delta_ub 18446744073.707097794s) v5 -- 0x558e818ee200 con 0x558e817ff800 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s delta_ub 18446744073.707097794s) v5 -- 0x558e818ee400 con 0x558e817ffc00 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s) v5 -- 0x558e818ee600 con 0x558e8181a000 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s) v5 -- 0x558e818ee800 con 0x558e8181a400 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s delta_ub 18446744073.698230848s) v5 -- 0x558e818eea00 con 0x558e817fe000 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s delta_ub 18446744073.698230848s) v5 -- 0x558e818eec00 con 0x558e817fe400 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s) v5 -- 0x558e818eee00 con 0x558e8181a800 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s) v5 -- 0x558e818ef000 con 0x558e8181ac00 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s) v5 -- 0x558e818ef200 con 0x558e817fe800 2019-12-07T19:33:41.251+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706752899s) v5 -- 0x558e818ef400 con 0x558e817fec00 2019-12-07T19:33:41.251+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6806/37077 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.723935495s delta_ub 0.017182596s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e817ff400 2019-12-07T19:33:41.251+0000 7fe2e6d3a700 25 osd.0 13 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 0.000000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.252305+0000 2019-12-07T19:33:41.251+0000 7fe2e6d3a700 10 osd.0 13 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.251+0000 7fe2e6d3a700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016938323s,0.017182596s]) 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6816/37076 1 ==== osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.709780695s delta_ub 0.003027796s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118fc00 con 0x558e817ff800 2019-12-07T19:33:41.251+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 1 ==== osd_map(14..14 src has 1..14) v4 ==== 580+0+0 (unknown 0 0 0) 0x558e817bbd40 con 0x558e818d6000 2019-12-07T19:33:41.251+0000 7fe2d4a80700 20 osd.0 13 OSD::ms_dispatch: osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.251+0000 7fe2d4a80700 10 osd.0 13 do_waiters -- start 2019-12-07T19:33:41.251+0000 7fe2d4a80700 10 osd.0 13 do_waiters -- finish 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 25 osd.0 13 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.252409+0000 last_rx_front 0.000000 2019-12-07T19:33:41.251+0000 7fe2d4a80700 20 osd.0 13 _dispatch 0x558e817bbd40 osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.251+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6814/37076 1 ==== osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.709828335s delta_ub 0.003075436s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e817ffc00 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 10 osd.0 13 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 0 -> 13 (as per caller) 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002679695s,0.003027796s]) 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6806/36936 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.718547470s delta_ub 0.011794571s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118fc00 con 0x558e817fe000 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 25 osd.0 13 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.252442+0000 last_rx_front 0.000000 2019-12-07T19:33:41.251+0000 7fe2d4a80700 3 osd.0 13 handle_osd_map epochs [14,14], i have 13, src has [1,14] 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 10 osd.0 13 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.251+0000 7fe2d4a80700 10 osd.0 13 handle_osd_map got inc map for epoch 14 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011413137s,0.011794571s]) 2019-12-07T19:33:41.251+0000 7fe2e753b700 25 osd.0 13 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252409+0000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.252456+0000 2019-12-07T19:33:41.251+0000 7fe2e753b700 25 osd.0 13 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:41.252061+0000) and older pending ping(s) 2019-12-07T19:33:41.251+0000 7fe2e753b700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002680383s,0.003075436s]) 2019-12-07T19:33:41.251+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:30f0e826:::osdmap.13:0# 0x0~0 2019-12-07T19:33:41.251+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1168 size 0x1168 (4456) 2019-12-07T19:33:41.251+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:41.251+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d1500 blob([0x1d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e811d16c0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6828/36939 1 ==== osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.687745086s delta_ub 18446744073.690543803s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118fc00 con 0x558e817fec00 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 25 osd.0 13 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 0.000000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.252501+0000 2019-12-07T19:33:41.251+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d1810 blob([!~1000,0x1e000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x558e811d1a40 sbid 0x0)) need 0x1000~168 cache has 0x[1000~168] 2019-12-07T19:33:41.251+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:30f0e826:::osdmap.13:0# 0x0~1168 = 4456 2019-12-07T19:33:41.251+0000 7fe2d4a80700 10 osd.0 13 add_map_bl 13 4456 bytes 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] conn(0x558e818fe000 0x558e818e5080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.7 up_from 0 peer_clock_delta [18446744073.690103943s,18446744073.690543803s]) 2019-12-07T19:33:41.251+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6808/37077 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.724053050s delta_ub 0.017300151s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee400 con 0x558e817ff000 2019-12-07T19:33:41.251+0000 7fe2e753b700 25 osd.0 13 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.252531+0000 last_rx_front 2019-12-07T19:33:41.252305+0000 2019-12-07T19:33:41.251+0000 7fe2e753b700 25 osd.0 13 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:41.252061+0000) and older pending ping(s) 2019-12-07T19:33:41.251+0000 7fe2e753b700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016830248s,0.017300151s]) 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6812/36938 1 ==== osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.701577844s delta_ub 18446744073.704376561s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118ea00 con 0x558e8181a400 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 25 osd.0 13 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 0.000000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.252600+0000 2019-12-07T19:33:41.251+0000 7fe2d4a80700 20 osd.0 13 got_full_map 14, nothing requested 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558e818fe400 0x558e818e5600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.4 up_from 0 peer_clock_delta [18446744073.703837339s,18446744073.704376561s]) 2019-12-07T19:33:41.251+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6804/36936 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.718609422s delta_ub 0.011856523s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee400 con 0x558e817fe400 2019-12-07T19:33:41.251+0000 7fe2e753b700 25 osd.0 13 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252442+0000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.252626+0000 2019-12-07T19:33:41.251+0000 7fe2e753b700 25 osd.0 13 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:41.252061+0000) and older pending ping(s) 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e818fe800 0x558e818e5b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.251+0000 7fe2d4a80700 20 osd.0 13 handle_osd_map pg_num_history pg_num_history(e14 pg_nums {} deleted_pools ) 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e818fec00 0x558e818e6100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.251+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps purged_snaps {14={}} 2019-12-07T19:33:41.251+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:41.251+0000 7fe2d4a80700 10 osd.0 13 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e14 [1,14] lci=[13,14]) 2019-12-07T19:33:41.251+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e818ff000 0x558e818e6680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:41.251+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e817de300 seq 7 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c3e3972f:::inc_osdmap.14:0# key 0x7f7fffffffffffffffc3e397'/!inc_osdmap.14!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:41.252+0000 7fe2e753b700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011291624s,0.011856523s]) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:41.252+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e818fec00 0x558e818e6100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.252+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6831/37078 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.709209801s delta_ub 0.002456902s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee400 con 0x558e81749800 2019-12-07T19:33:41.252+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c3e3972f:::inc_osdmap.14:0# 0x0~218 2019-12-07T19:33:41.252+0000 7fe2e753b700 25 osd.0 13 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.252746+0000 last_rx_front 0.000000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1055 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c3e3972f:::inc_osdmap.14:0# 0x0~218 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:41.252+0000 7fe2e753b700 10 osd.0 13 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~218 2019-12-07T19:33:41.252+0000 7fe2e753b700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.001771781s,0.002456902s]) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xde8 on front/back, now 0x0~1000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e817de300 1 blobs 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x1f000~1000/1000,1000,0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x1f000~1000] 2019-12-07T19:33:41.252+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] conn(0x558e818fe000 0x558e818e5080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81866380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81866fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81866380 blob([0x1f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81866fc0 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.blob(0x558e81866380) get_ref 0x0~218 Blob(0x558e81866380 blob([0x1f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81866fc0 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.blob(0x558e81866380) get_ref init 0x1000, 1000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~218: 0x0~218 Blob(0x558e81866380 blob([0x1f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x218) SharedBlob(0x558e81866fc0 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81866fd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x218 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191a108) dirty_range mark inline shard dirty 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c3e3972f:::inc_osdmap.14:0# 0x0~218 = 0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3b30e826:::osdmap.14:0# key 0x7f7fffffffffffffff3b30e8'&!osdmap.14!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3b30e826:::osdmap.14:0# 0x0~1168 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1056 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3b30e826:::osdmap.14:0# 0x0~1168 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:41.252+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e818ff000 0x558e818e6680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~168 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xe98 on front/back, now 0x0~1000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e817de300 2 blobs 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x20000~2000/1000,2000,0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x20000~2000] 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81866d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e818670a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:41.252+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.6 v2:172.21.15.109:6822/36937 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706852724s delta_ub 0.000099825s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e8181a800 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81866d90 blob([0x20000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e818670a0 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.blob(0x558e81866d90) get_ref 0x0~1000 Blob(0x558e81866d90 blob([0x20000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e818670a0 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.blob(0x558e81866d90) get_ref init 0x1000, 1000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e81866d90 blob([0x20000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e818670a0 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2e753b700 25 osd.0 13 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.252854+0000 last_rx_front 0.000000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e818670b8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e818672d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81867340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e818672d0 blob([!~1000,0x21000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81867340 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2e7d3c700 10 osd.0 13 ms_handle_authentication new session 0x558e818c36c0 con 0x558e818fec00 entity osd.6 addr 2019-12-07T19:33:41.252+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818ffc00 0x558e818e7180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).connect 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.blob(0x558e818672d0) get_ref 0x1000~168 Blob(0x558e818672d0 blob([!~1000,0x21000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e81867340 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.blob(0x558e818672d0) get_ref init 0x2000, 1000 2019-12-07T19:33:41.252+0000 7fe2e753b700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.6 up_from 0 peer_clock_delta [18446744073.708858650s,0.000099825s]) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~168: 0x1000~168 Blob(0x558e818672d0 blob([!~1000,0x21000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x558e81867340 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81867358 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1168 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191a388) dirty_range mark inline shard dirty 2019-12-07T19:33:41.252+0000 7fe2e7d3c700 10 osd.0 13 ms_handle_authentication session 0x558e818c36c0 osd.6 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3b30e826:::osdmap.14:0# 0x0~1168 = 0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~12 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfee 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~12: 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e817de300 0 blobs 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818fec00 0x558e818e6100 crc :-1 s=SESSION_ACCEPTING pgs=7 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, replacing existing=0x558e818ffc00 socket by this connection's socket 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e817de300 0 blobs 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:41.252+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818fec00 0x558e818e6100 crc :-1 s=SESSION_ACCEPTING pgs=7 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e817de300 cost 22453 (4 ios * 4000 + 6453 bytes) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e817de300 onodes 0x558e80251b80,0x558e817b9180,0x558e8191a000,0x558e8191a280 shared_blobs 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191a108) update #-1:c3e3972f:::inc_osdmap.14:0# 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191a108) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c3e3972f:::inc_osdmap.14:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191a388) update #-1:3b30e826:::osdmap.14:0# 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191a388) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3b30e826:::osdmap.14:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e817de300 allocated 0x[1f000~3000] released 0x[] 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 freelist allocate 0x1f000~3000 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:41.252+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] reap_dead start 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de300 prepare 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e817de300 2019-12-07T19:33:41.252+0000 7fe2e7d3c700 10 osd.0 13 new session (incoming)0x558e818c3860 con=0x558e818ffc00 addr=v2:172.21.15.109:6818/36937 must have raced with connect 2019-12-07T19:33:41.252+0000 7fe2e753b700 10 osd.0 13 ms_handle_authentication new session 0x558e818c31e0 con 0x558e818ff000 entity osd.7 addr 2019-12-07T19:33:41.252+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818ffc00 0x558e818e7180 unknown :-1 s=READY pgs=7 cs=0 l=0 rx=0 tx=0).ready entity=osd.6 client_cookie=66df44187c63ea59 server_cookie=2175ee4a8507a6a8 in_seq=0 out_seq=0 2019-12-07T19:33:41.252+0000 7fe2e753b700 10 osd.0 13 ms_handle_authentication session 0x558e818c31e0 osd.7 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.252+0000 7fe2d4a80700 2 osd.0 13 ms_handle_reset con 0x558e818fec00 session 0x558e818c36c0 2019-12-07T19:33:41.252+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 1 ==== osd_map(14..14 src has 1..14) v4 ==== 580+0+0 (unknown 0 0 0) 0x558e81908000 con 0x558e818d6800 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 osd.0 13 OSD::ms_dispatch: osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 osd.0 13 do_waiters -- start 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 osd.0 13 do_waiters -- finish 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 osd.0 13 _dispatch 0x558e81908000 osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.252+0000 7fe2d4a80700 3 osd.0 13 handle_osd_map epochs [14,14], i have 14, src has [1,14] 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 osd.0 13 no new maps here, dropping 2019-12-07T19:33:41.252+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558e818fe400 0x558e818e5600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.252+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de300 aio_wait 2019-12-07T19:33:41.252+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e817de300 2019-12-07T19:33:41.252+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de300 io_done 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.6 v2:172.21.15.109:6820/36937 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.706775745s delta_ub 0.000022846s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e8181ac00 2019-12-07T19:33:41.252+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 1 ==== osd_map(14..14 src has 1..14) v4 ==== 580+0+0 (unknown 0 0 0) 0x558e80238fc0 con 0x558e818d7c00 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 25 osd.0 13 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252854+0000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.253269+0000 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 25 osd.0 13 handle_osd_ping got all replies from osd.6 , erase pending ping(sent at 2019-12-07T19:33:41.252061+0000) and older pending ping(s) 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:41.252+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] conn(0x558e818ff000 0x558e818e6680 crc :-1 s=SESSION_ACCEPTING pgs=7 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, replacing existing=0x558e818fe000 socket by this connection's socket 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 osd.0 13 OSD::ms_dispatch: osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 osd.0 13 do_waiters -- start 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 osd.0 13 do_waiters -- finish 2019-12-07T19:33:41.252+0000 7fe2d4a80700 20 osd.0 13 _dispatch 0x558e80238fc0 osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 10 osd.0 13 maybe_share_map con 0x558e818ffc00 v2:172.21.15.109:6818/36937 map epoch 0 -> 14 (as per caller) 2019-12-07T19:33:41.252+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] conn(0x558e818ff000 0x558e818e6680 crc :-1 s=SESSION_ACCEPTING pgs=7 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.6 up_from 0 peer_clock_delta [18446744073.708366599s,0.000022846s]) 2019-12-07T19:33:41.252+0000 7fe2d4a80700 3 osd.0 13 handle_osd_map epochs [14,14], i have 14, src has [1,14] 2019-12-07T19:33:41.252+0000 7fe2d4a80700 10 osd.0 13 no new maps here, dropping 2019-12-07T19:33:41.252+0000 7fe2d4a80700 2 osd.0 13 ms_handle_reset con 0x558e818ff000 session 0x558e818c31e0 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6814/36938 1 ==== osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.701469379s delta_ub 18446744073.704268096s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e8181a000 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 25 osd.0 13 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.253306+0000 last_rx_front 2019-12-07T19:33:41.252600+0000 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 25 osd.0 13 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:41.252061+0000) and older pending ping(s) 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.4 up_from 0 peer_clock_delta [18446744073.703022881s,18446744073.704268096s]) 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6830/36939 1 ==== osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.687944339s delta_ub 18446744073.690743056s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e817fe800 2019-12-07T19:33:41.252+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.6 172.21.15.109:0/36937 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s delta_ub 0.000099825s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e802a4c00 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 25 osd.0 13 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 0.000000 -> 2019-12-07T19:33:41.253337+0000 last_rx_front 2019-12-07T19:33:41.252501+0000 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 25 osd.0 13 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:33:41.252061+0000) and older pending ping(s) 2019-12-07T19:33:41.252+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8168 < min_flush_size 524288 2019-12-07T19:33:41.252+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8799 < min_flush_size 524288 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.7 up_from 0 peer_clock_delta [18446744073.689466447s,18446744073.690743056s]) 2019-12-07T19:33:41.252+0000 7fe2e753b700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709167847s,0.000099825s]) 2019-12-07T19:33:41.252+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.708049322s delta_ub 0.000383769s) v5 -- 0x558e818ee600 con 0x558e802a4c00 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8191a000 had 1 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8191a280 had 1 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:33:41.252+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6827/37078 1 ==== osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.252061+0000/4.706752899s send_stamp 4.709576414s delta_ub 0.002823515s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e81749c00 2019-12-07T19:33:41.252+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 25 osd.0 13 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252746+0000 last_rx_front 0.000000 -> 2019-12-07T19:33:41.253382+0000 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 25 osd.0 13 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:41.252061+0000) and older pending ping(s) 2019-12-07T19:33:41.252+0000 7fe2e753b700 10 osd.0 13 new session (incoming)0x558e818c2ea0 con=0x558e818fe000 addr=v2:172.21.15.109:6826/36939 must have raced with connect 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.001501845s,0.002823515s]) 2019-12-07T19:33:41.252+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8870 < min_flush_size 524288 2019-12-07T19:33:41.252+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8870 < min_flush_size 524288 2019-12-07T19:33:41.252+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] conn(0x558e818fe000 0x558e818e5080 unknown :-1 s=READY pgs=7 cs=0 l=0 rx=0 tx=0).ready entity=osd.7 client_cookie=613b45576f9f3c20 server_cookie=9a36ce1d1f5316c2 in_seq=0 out_seq=0 2019-12-07T19:33:41.252+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:41.252+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x6018~22a6 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:41.252+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x6018 0x6018~22a6 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x6018 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x18 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x2be and padding block with 0xd42 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e818fe800 0x558e818e5b80 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:41.252+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.6 172.21.15.109:0/36937 1 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.707665553s delta_ub 0.000099825s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118fc00 con 0x558e8182cc00 2019-12-07T19:33:41.252+0000 7fe2e753b700 20 osd.0 13 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709061380s,0.000099825s]) 2019-12-07T19:33:41.252+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e13 up_from 13 ping_stamp 2019-12-07T19:33:41.253160+0000/4.707665553s send_stamp 4.708155789s delta_ub 0.000490236s) v5 -- 0x558e818ee600 con 0x558e8182cc00 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x82be 2019-12-07T19:33:41.252+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:41.252+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:41.252+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication new session 0x558e818e8000 con 0x558e818fe800 entity osd.4 addr 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 1 in 0.000324832s (0.000065288s flush + 0.000259544s kv commit) 2019-12-07T19:33:41.252+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:41.252+0000 7fe2e6d3a700 10 osd.0 13 ms_handle_authentication session 0x558e818e8000 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e817de300> 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <0x558e8107b2c0> 2019-12-07T19:33:41.252+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de300 kv_submitted 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e817de300 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e817de300 osr 0x558e8107a500 2019-12-07T19:33:41.252+0000 7fe2d7285700 10 osd.0 13 _committed_osd_maps 14..14 2019-12-07T19:33:41.252+0000 7fe2d7285700 10 osd.0 13 advance to epoch 14 (<= last 14 <= newest_map 14) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de000) prepare_write seq 5 0x1b000~1000 crc 4c6c8a7 2019-12-07T19:33:41.252+0000 7fe2d7285700 7 osd.0 14 consume_map version 14 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de000) prepare_write seq 5 0x2000~1000 crc 19be0fd3 2019-12-07T19:33:41.252+0000 7fe2d7285700 20 osd.0 14 prune_pg_created 2019-12-07T19:33:41.252+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 deferred_cleanup 2019-12-07T19:33:41.252+0000 7fe2d7285700 10 osd.0:0.consume_map 14 (was 13) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c300 onodes 0x558e80251b80 2019-12-07T19:33:41.252+0000 7fe2d7285700 10 osd.0:1.consume_map 14 (was 13) 2019-12-07T19:33:41.252+0000 7fe2d7285700 10 osd.0:2.consume_map 14 (was 13) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c300 done 2019-12-07T19:33:41.252+0000 7fe2d7285700 10 osd.0:3.consume_map 14 (was 13) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c900 deferred_cleanup 2019-12-07T19:33:41.252+0000 7fe2d7285700 10 osd.0:4.consume_map 14 (was 13) 2019-12-07T19:33:41.252+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c300 [] 2019-12-07T19:33:41.252+0000 7fe2d7285700 10 osd.0:5.consume_map 14 (was 13) 2019-12-07T19:33:41.252+0000 7fe2d7285700 10 osd.0:6.consume_map 14 (was 13) 2019-12-07T19:33:41.252+0000 7fe2d7285700 10 osd.0:7.consume_map 14 (was 13) 2019-12-07T19:33:41.252+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:41.252+0000 7fe2d7285700 10 osd.0 14 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:41.252+0000 7fe2d7285700 7 osd.0 14 activate_map version 14 2019-12-07T19:33:41.252+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 deferred_cleanup 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c900 onodes 0x558e80251b80 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c900 done 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c000 deferred_cleanup 2019-12-07T19:33:41.252+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c900 [] 2019-12-07T19:33:41.252+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:41.252+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 deferred_cleanup 2019-12-07T19:33:41.252+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558e818fe400 0x558e818e5600 crc :-1 s=START_CONNECT pgs=0 cs=0 l=0 rx=0 tx=0)._fault waiting 15.000000 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c000 onodes 0x558e80251b80,0x558e8125c500,0x558e8125d180,0x558e8125d400,0x558e8125d680,0x558e8125d900,0x558e8125db80,0x558e81796000,0x558e81796280,0x558e81796500,0x558e81796780,0x558e81796a00,0x558e81796c80,0x558e81796f00,0x558e81797180,0x558e81797400,0x558e81797680,0x558e81797900,0x558e81797b80,0x558e817b8000,0x558e817b8280,0x558e817b8500,0x558e817b8780,0x558e817b8a00,0x558e817b8c80,0x558e817b8f00,0x558e817b9180 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d1d68 in 0x558e80219340) _finish_write discard buffer(0x558e81170750 space 0x558e811d1d68 0x0~1000 writing nocache) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d1e48 in 0x558e80219340) _finish_write discard buffer(0x558e811706c0 space 0x558e811d1e48 0x0~1000 writing nocache) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81788018 in 0x558e80219340) _finish_write discard buffer(0x558e81170630 space 0x558e81788018 0x0~1000 writing nocache) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81788168 in 0x558e80219340) _finish_write discard buffer(0x558e81170480 space 0x558e81788168 0x0~1000 writing nocache) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81788398 in 0x558e80219340) _finish_write discard buffer(0x558e811705a0 space 0x558e81788398 0x0~1000 writing nocache) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817884e8 in 0x558e80219340) _finish_write discard buffer(0x558e81170510 space 0x558e817884e8 0x0~1000 writing nocache) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81788638 in 0x558e80219340) _finish_write discard buffer(0x558e811707e0 space 0x558e81788638 0x0~1000 writing nocache) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81788788 in 0x558e80219340) _finish_write discard buffer(0x558e81170870 space 0x558e81788788 0x0~1000 writing nocache) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817888d8 in 0x558e80219340) _finish_write discard buffer(0x558e81170900 space 0x558e817888d8 0x0~1000 writing nocache) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81788a28 in 0x558e80219340) _finish_write discard buffer(0x558e81170990 space 0x558e81788a28 0x0~1000 writing nocache) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81788b78 in 0x558e80219340) _finish_write discard buffer(0x558e81170a20 space 0x558e81788b78 0x0~1000 writing nocache) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81788cc8 in 0x558e80219340) _finish_write discard buffer(0x558e81170ab0 space 0x558e81788cc8 0x0~1000 writing nocache) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81788e18 in 0x558e80219340) _finish_write discard buffer(0x558e81170b40 space 0x558e81788e18 0x0~1000 writing nocache) 2019-12-07T19:33:41.252+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81788f68 in 0x558e80219340) _finish_write discard buffer(0x558e81170bd0 space 0x558e81788f68 0x0~1000 writing nocache) 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817890b8 in 0x558e80219340) _finish_write discard buffer(0x558e81170c60 space 0x558e817890b8 0x0~1000 writing nocache) 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81789208 in 0x558e80219340) _finish_write discard buffer(0x558e81170cf0 space 0x558e81789208 0x0~1000 writing nocache) 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81789358 in 0x558e80219340) _finish_write discard buffer(0x558e81170d80 space 0x558e81789358 0x0~1000 writing nocache) 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817894a8 in 0x558e80219340) _finish_write discard buffer(0x558e81170e10 space 0x558e817894a8 0x0~1000 writing nocache) 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817895f8 in 0x558e80219340) _finish_write discard buffer(0x558e81170ea0 space 0x558e817895f8 0x0~1000 writing nocache) 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81789748 in 0x558e80219340) _finish_write discard buffer(0x558e81170f30 space 0x558e81789748 0x0~1000 writing nocache) 2019-12-07T19:33:41.253+0000 7fe2cc26f700 20 osd.0 14 agent_entry empty queue 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81789898 in 0x558e80219340) _finish_write discard buffer(0x558e81170fc0 space 0x558e81789898 0x0~1000 writing nocache) 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817899e8 in 0x558e80219340) _finish_write discard buffer(0x558e81171050 space 0x558e817899e8 0x0~1000 writing nocache) 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81789b38 in 0x558e80219340) _finish_write discard buffer(0x558e811710e0 space 0x558e81789b38 0x0~1000 writing nocache) 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81789c88 in 0x558e80219340) _finish_write discard buffer(0x558e81171170 space 0x558e81789c88 0x0~1000 writing nocache) 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c000 done 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e817de000 deferred_cleanup 2019-12-07T19:33:41.253+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c000 [] 2019-12-07T19:33:41.253+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:41.253+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558e818fe800 0x558e818e5b80 crc :-1 s=SESSION_ACCEPTING pgs=6 cs=0 l=0 rx=0 tx=0).handle_existing_connection connection race detected, replacing existing=0x558e818fe400 socket by this connection's socket 2019-12-07T19:33:41.253+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de000 deferred_cleanup 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e817de000 onodes 0x558e80251b80,0x558e817b9180,0x558e817b9400,0x558e817b9680 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d1198 in 0x558e80219340) _finish_write discard buffer(0x558e81171320 space 0x558e811d1198 0x0~1000 writing nocache) 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d16d8 in 0x558e80219340) _finish_write discard buffer(0x558e811713b0 space 0x558e811d16d8 0x0~1000 writing nocache) 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d1a58 in 0x558e80219340) _finish_write discard buffer(0x558e81171440 space 0x558e811d1a58 0x1000~1000 writing nocache) 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e817de000 done 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e817de300 deferred_queued 2019-12-07T19:33:41.253+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e817de000 [] 2019-12-07T19:33:41.253+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:41.253+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558e818fe800 0x558e818e5b80 crc :-1 s=SESSION_ACCEPTING pgs=6 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:41.253+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:41.253+0000 7fe2d4a80700 2 osd.0 14 ms_handle_reset con 0x558e818fe800 session 0x558e818e8000 2019-12-07T19:33:41.253+0000 7fe2e6d3a700 10 osd.0 14 new session (incoming)0x558e818e81a0 con=0x558e818fe400 addr=v2:172.21.15.109:6810/36938 must have raced with connect 2019-12-07T19:33:41.253+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558e818fe400 0x558e818e5600 unknown :-1 s=READY pgs=6 cs=0 l=0 rx=0 tx=0).ready entity=osd.4 client_cookie=35acc3f1ec36344f server_cookie=b1c9bc281f9b9f1e in_seq=0 out_seq=0 2019-12-07T19:33:41.253+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 1 ==== osd_map(14..14 src has 1..14) v4 ==== 580+0+0 (unknown 0 0 0) 0x558e80239440 con 0x558e818ffc00 2019-12-07T19:33:41.253+0000 7fe2d4a80700 20 osd.0 14 OSD::ms_dispatch: osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.253+0000 7fe2d4a80700 10 osd.0 14 do_waiters -- start 2019-12-07T19:33:41.253+0000 7fe2d4a80700 10 osd.0 14 do_waiters -- finish 2019-12-07T19:33:41.253+0000 7fe2d4a80700 20 osd.0 14 _dispatch 0x558e80239440 osd_map(14..14 src has 1..14) v4 2019-12-07T19:33:41.253+0000 7fe2d4a80700 3 osd.0 14 handle_osd_map epochs [14,14], i have 14, src has [1,14] 2019-12-07T19:33:41.253+0000 7fe2d4a80700 10 osd.0 14 no new maps here, dropping 2019-12-07T19:33:41.256+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.257+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.4 172.21.15.109:0/36938 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.257684+0000/4.706911153s send_stamp 4.706911153s delta_ub 18446744073.704376561s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e8182d000 2019-12-07T19:33:41.257+0000 7fe2e6d3a700 20 osd.0 14 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703805363s,18446744073.704376561s]) 2019-12-07T19:33:41.257+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.257684+0000/4.706911153s send_stamp 4.712657406s delta_ub 0.005746253s) v5 -- 0x558e818a9c00 con 0x558e8182d000 2019-12-07T19:33:41.257+0000 7fe2e6d3a700 10 osd.0 14 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 0 -> 13 (as per caller) 2019-12-07T19:33:41.257+0000 7fe2e6d3a700 10 osd.0 14 send_incremental_map 13 -> 14 to 0x558e818fe400 v2:172.21.15.109:6810/36938 2019-12-07T19:33:41.257+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.4 172.21.15.109:0/36938 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.257684+0000/4.706911153s send_stamp 4.706911153s delta_ub 18446744073.704376561s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118fc00 con 0x558e8181bc00 2019-12-07T19:33:41.257+0000 7fe2e6d3a700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c3e3972f:::inc_osdmap.14:0# 0x0~0 2019-12-07T19:33:41.257+0000 7fe2e6d3a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~218 size 0x218 (536) 2019-12-07T19:33:41.257+0000 7fe2e6d3a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:41.257+0000 7fe2e6d3a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866380 blob([0x1f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x218) SharedBlob(0x558e81866fc0 sbid 0x0)) need 0x0~218 cache has 0x[0~218] 2019-12-07T19:33:41.257+0000 7fe2e6d3a700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c3e3972f:::inc_osdmap.14:0# 0x0~218 = 536 2019-12-07T19:33:41.257+0000 7fe2e6d3a700 10 osd.0 14 add_map_inc_bl 14 536 bytes 2019-12-07T19:33:41.257+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(14..14 src has 1..14) v4 -- 0x558e80238fc0 con 0x558e818fe400 2019-12-07T19:33:41.257+0000 7fe2e6d3a700 10 osd.0 14 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 13 -> 14 (shared) 2019-12-07T19:33:41.257+0000 7fe2e753b700 20 osd.0 14 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703654448s,18446744073.704376561s]) 2019-12-07T19:33:41.257+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.257684+0000/4.706911153s send_stamp 4.712808321s delta_ub 0.005897168s) v5 -- 0x558e818a9000 con 0x558e8181bc00 2019-12-07T19:33:41.270+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.7 172.21.15.109:0/36939 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.690743056s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e81749000 2019-12-07T19:33:41.270+0000 7fe2e6d3a700 20 osd.0 14 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690014311s,18446744073.690743056s]) 2019-12-07T19:33:41.270+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.726157852s delta_ub 0.019537305s) v5 -- 0x558e818a9c00 con 0x558e81749000 2019-12-07T19:33:41.270+0000 7fe2e6d3a700 10 osd.0 14 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 0 -> 13 (as per caller) 2019-12-07T19:33:41.270+0000 7fe2e6d3a700 10 osd.0 14 send_incremental_map 13 -> 14 to 0x558e818fe000 v2:172.21.15.109:6826/36939 2019-12-07T19:33:41.270+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(14..14 src has 1..14) v4 -- 0x558e80238fc0 con 0x558e818fe000 2019-12-07T19:33:41.270+0000 7fe2e6d3a700 10 osd.0 14 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 13 -> 14 (shared) 2019-12-07T19:33:41.270+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.7 172.21.15.109:0/36939 1 ==== osd_ping(ping e13 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.706620547s delta_ub 18446744073.690743056s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e81748c00 2019-12-07T19:33:41.270+0000 7fe2e6d3a700 20 osd.0 14 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.689858486s,18446744073.690743056s]) 2019-12-07T19:33:41.270+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.271181+0000/4.706620547s send_stamp 4.726313677s delta_ub 0.019693130s) v5 -- 0x558e818a8a00 con 0x558e81748c00 2019-12-07T19:33:41.306+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.356+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.407+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.457+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.507+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.532+0000 7fe2e2ca2700 10 osd.0 14 tick_without_osd_lock 2019-12-07T19:33:41.532+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffde000/0x40000000/0x1680000000, data 0x85ce/0x20000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:41.532+0000 7fe2e2ca2700 25 osd.0 14 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252746+0000 last_rx_front 2019-12-07T19:33:41.253382+0000 2019-12-07T19:33:41.532+0000 7fe2e2ca2700 25 osd.0 14 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252531+0000 last_rx_front 2019-12-07T19:33:41.252305+0000 2019-12-07T19:33:41.532+0000 7fe2e2ca2700 25 osd.0 14 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252409+0000 last_rx_front 2019-12-07T19:33:41.252456+0000 2019-12-07T19:33:41.532+0000 7fe2e2ca2700 25 osd.0 14 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.253306+0000 last_rx_front 2019-12-07T19:33:41.252600+0000 2019-12-07T19:33:41.532+0000 7fe2e2ca2700 25 osd.0 14 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252442+0000 last_rx_front 2019-12-07T19:33:41.252626+0000 2019-12-07T19:33:41.532+0000 7fe2e2ca2700 25 osd.0 14 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252854+0000 last_rx_front 2019-12-07T19:33:41.253269+0000 2019-12-07T19:33:41.532+0000 7fe2e2ca2700 25 osd.0 14 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.253337+0000 last_rx_front 2019-12-07T19:33:41.252501+0000 2019-12-07T19:33:41.532+0000 7fe2e2ca2700 20 osd.0 14 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:41.532+0000 7fe2e2ca2700 10 osd.0 14 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:41.532+0000 7fe2e2ca2700 20 osd.0 14 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:41.532+0000 7fe2e2ca2700 10 osd.0 14 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:41.557+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:33:41.557+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:33:41.557+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:41.557+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:41.557+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:41.557+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:41.557+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:41.557+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:41.557+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:41.557+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:41.557+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:41.557+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:41.557+0000 7fe2dc28f700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:33:41.557+0000 7fe2dc28f700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258771 2019-12-07T19:33:41.557+0000 7fe2dc28f700 5 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.607+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.626+0000 7fe2e4535700 10 osd.0 14 tick 2019-12-07T19:33:41.626+0000 7fe2e4535700 10 osd.0 14 do_waiters -- start 2019-12-07T19:33:41.626+0000 7fe2e4535700 10 osd.0 14 do_waiters -- finish 2019-12-07T19:33:41.626+0000 7fe2e4535700 20 osd.0 14 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:41.657+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.707+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.740+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.5 172.21.15.109:0/36936 2 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.740997+0000/5.207320920s send_stamp 5.207320920s delta_ub 0.011856523s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef400 con 0x558e8181b800 2019-12-07T19:33:41.740+0000 7fe2e7d3c700 20 osd.0 14 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011197547s,0.011856523s]) 2019-12-07T19:33:41.740+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.740997+0000/5.207320920s send_stamp 5.196123373s delta_ub 18446744073.698354069s) v5 -- 0x558e818ee400 con 0x558e8181b800 2019-12-07T19:33:41.740+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.5 172.21.15.109:0/36936 2 ==== osd_ping(ping e14 up_from 13 ping_stamp 2019-12-07T19:33:41.740997+0000/5.207320920s send_stamp 5.207320920s delta_ub 0.011856523s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef400 con 0x558e8182c800 2019-12-07T19:33:41.740+0000 7fe2e7d3c700 20 osd.0 14 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011093450s,0.011856523s]) 2019-12-07T19:33:41.740+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:41.740997+0000/5.207320920s send_stamp 5.196227470s delta_ub 18446744073.698458166s) v5 -- 0x558e818eea00 con 0x558e8182c800 2019-12-07T19:33:41.757+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.808+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.858+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.908+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:41.958+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.008+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.058+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.108+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.158+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.209+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.259+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.309+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 37904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.357+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.4 172.21.15.109:0/36938 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:42.358166+0000/5.807392265s send_stamp 5.807392265s delta_ub 18446744073.704324246s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e8181bc00 2019-12-07T19:33:42.357+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.4 172.21.15.109:0/36938 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:42.358166+0000/5.807392265s send_stamp 5.807392265s delta_ub 18446744073.704324246s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e8182d000 2019-12-07T19:33:42.357+0000 7fe2e753b700 20 osd.0 14 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703782112s,18446744073.704324246s]) 2019-12-07T19:33:42.357+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:42.358166+0000/5.807392265s send_stamp 5.813161769s delta_ub 0.005769504s) v5 -- 0x558e818a9000 con 0x558e8181bc00 2019-12-07T19:33:42.357+0000 7fe2e753b700 10 osd.0 14 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:42.357+0000 7fe2e6d3a700 20 osd.0 14 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703746043s,18446744073.704324246s]) 2019-12-07T19:33:42.357+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e14 up_from 13 ping_stamp 2019-12-07T19:33:42.358166+0000/5.807392265s send_stamp 5.813197838s delta_ub 0.005805573s) v5 -- 0x558e818a8a00 con 0x558e8182d000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 1 ==== osd_map(15..15 src has 1..15) v4 ==== 380+0+0 (unknown 0 0 0) 0x558e817bbb00 con 0x558e818fe400 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 osd.0 14 OSD::ms_dispatch: osd_map(15..15 src has 1..15) v4 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 osd.0 14 do_waiters -- start 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 osd.0 14 do_waiters -- finish 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 osd.0 14 _dispatch 0x558e817bbb00 osd_map(15..15 src has 1..15) v4 2019-12-07T19:33:42.358+0000 7fe2d4a80700 3 osd.0 14 handle_osd_map epochs [15,15], i have 14, src has [1,15] 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 osd.0 14 handle_osd_map got inc map for epoch 15 2019-12-07T19:33:42.358+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3b30e826:::osdmap.14:0# 0x0~0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1168 size 0x1168 (4456) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866d90 blob([0x20000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e818670a0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e818672d0 blob([!~1000,0x21000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x558e81867340 sbid 0x0)) need 0x1000~168 cache has 0x[1000~168] 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3b30e826:::osdmap.14:0# 0x0~1168 = 4456 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 osd.0 14 add_map_bl 14 4456 bytes 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 osd.0 14 got_full_map 15, nothing requested 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 osd.0 14 handle_osd_map pg_num_history pg_num_history(e15 pg_nums {} deleted_pools ) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps purged_snaps {15={}} 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 osd.0 14 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e15 [1,15] lci=[13,15]) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e8106c300 seq 8 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c923972f:::inc_osdmap.15:0# key 0x7f7fffffffffffffffc92397'/!inc_osdmap.15!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c923972f:::inc_osdmap.15:0# 0x0~150 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1057 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c923972f:::inc_osdmap.15:0# 0x0~150 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~150 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xeb0 on front/back, now 0x0~1000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c300 1 blobs 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x22000~1000/1000,1000,0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x22000~1000] 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81866150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e818665b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81866150 blob([0x22000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e818665b0 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.blob(0x558e81866150) get_ref 0x0~150 Blob(0x558e81866150 blob([0x22000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e818665b0 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.blob(0x558e81866150) get_ref init 0x1000, 1000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~150: 0x0~150 Blob(0x558e81866150 blob([0x22000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x150) SharedBlob(0x558e818665b0 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e818665c8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x150 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191a608) dirty_range mark inline shard dirty 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c923972f:::inc_osdmap.15:0# 0x0~150 = 0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3630e826:::osdmap.15:0# key 0x7f7fffffffffffffff3630e8'&!osdmap.15!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3630e826:::osdmap.15:0# 0x0~1168 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1058 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3630e826:::osdmap.15:0# 0x0~1168 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~168 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xe98 on front/back, now 0x0~1000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c300 2 blobs 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x23000~2000/1000,2000,0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x23000~2000] 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81866b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81789f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81866b60 blob([0x23000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789f80 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.blob(0x558e81866b60) get_ref 0x0~1000 Blob(0x558e81866b60 blob([0x23000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81789f80 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.blob(0x558e81866b60) get_ref init 0x1000, 1000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e81866b60 blob([0x23000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e81789f80 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789f98 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e80f04380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e818676c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e80f04380 blob([!~1000,0x24000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e818676c0 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.blob(0x558e80f04380) get_ref 0x1000~168 Blob(0x558e80f04380 blob([!~1000,0x24000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e818676c0 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.blob(0x558e80f04380) get_ref init 0x2000, 1000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~168: 0x1000~168 Blob(0x558e80f04380 blob([!~1000,0x24000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x558e818676c0 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e818676d8 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1168 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191a888) dirty_range mark inline shard dirty 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3630e826:::osdmap.15:0# 0x0~1168 = 0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~12 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfee 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~12: 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c300 0 blobs 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c300 0 blobs 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c300 cost 22253 (4 ios * 4000 + 6253 bytes) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c300 onodes 0x558e80251b80,0x558e817b9180,0x558e8191a500,0x558e8191a780 shared_blobs 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191a608) update #-1:c923972f:::inc_osdmap.15:0# 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191a608) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c923972f:::inc_osdmap.15:0# is 46 (22 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191a888) update #-1:3630e826:::osdmap.15:0# 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191a888) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3630e826:::osdmap.15:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c300 allocated 0x[22000~3000] released 0x[] 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 freelist allocate 0x22000~3000 2019-12-07T19:33:42.358+0000 7fe2d4a80700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 prepare 2019-12-07T19:33:42.358+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8106c300 2019-12-07T19:33:42.358+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 aio_wait 2019-12-07T19:33:42.358+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c300 2019-12-07T19:33:42.358+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 io_done 2019-12-07T19:33:42.358+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:42.358+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:42.358+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:42.359+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8787 < min_flush_size 524288 2019-12-07T19:33:42.359+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:33:42.359+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:33:42.359+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8191a500 had 1 2019-12-07T19:33:42.359+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8191a780 had 1 2019-12-07T19:33:42.359+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:33:42.359+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:42.359+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:42.359+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8810 < min_flush_size 524288 2019-12-07T19:33:42.359+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8810 < min_flush_size 524288 2019-12-07T19:33:42.359+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:42.359+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x82be~226a to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:42.359+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x82be 0x82be~226a to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:42.359+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:42.359+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0x82be 2019-12-07T19:33:42.359+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x2be 2019-12-07T19:33:42.359+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:42.359+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x528 and padding block with 0xad8 2019-12-07T19:33:42.359+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xa528 2019-12-07T19:33:42.359+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:42.359+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:42.359+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:42.359+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:42.359+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000225500s (0.000064054s flush + 0.000161446s kv commit) 2019-12-07T19:33:42.359+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:42.359+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:42.359+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8106c300> 2019-12-07T19:33:42.359+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:42.359+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 kv_submitted 2019-12-07T19:33:42.359+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c300 2019-12-07T19:33:42.359+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e8106c300 osr 0x558e8107a500 2019-12-07T19:33:42.359+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de000) _discard drop 5 0x1b000~1000 2019-12-07T19:33:42.359+0000 7fe2d7285700 10 osd.0 14 _committed_osd_maps 15..15 2019-12-07T19:33:42.359+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de000) prepare_write seq 6 0x1b000~1000 crc 1e19645a 2019-12-07T19:33:42.359+0000 7fe2d7285700 10 osd.0 14 advance to epoch 15 (<= last 15 <= newest_map 15) 2019-12-07T19:33:42.359+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de000) _discard drop 5 0x2000~1000 2019-12-07T19:33:42.359+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de000) prepare_write seq 6 0x2000~1000 crc a2851bee 2019-12-07T19:33:42.359+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:42.359+0000 7fe2d7285700 7 osd.0 15 consume_map version 15 2019-12-07T19:33:42.359+0000 7fe2d7285700 20 osd.0 15 prune_pg_created 2019-12-07T19:33:42.359+0000 7fe2d7285700 10 osd.0:0.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7fe2d7285700 10 osd.0:1.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7fe2d7285700 10 osd.0:2.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7fe2d7285700 10 osd.0:3.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7fe2d7285700 10 osd.0:4.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7fe2d7285700 10 osd.0:5.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7fe2d7285700 10 osd.0:6.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7fe2d7285700 10 osd.0:7.consume_map 15 (was 14) 2019-12-07T19:33:42.359+0000 7fe2d7285700 10 osd.0 15 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:42.359+0000 7fe2d7285700 7 osd.0 15 activate_map version 15 2019-12-07T19:33:42.359+0000 7fe2cc26f700 20 osd.0 15 agent_entry empty queue 2019-12-07T19:33:42.359+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.409+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.459+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.488+0000 7fe2e2ca2700 10 osd.0 15 tick_without_osd_lock 2019-12-07T19:33:42.488+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:42.488+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252746+0000 last_rx_front 2019-12-07T19:33:41.253382+0000 2019-12-07T19:33:42.488+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252531+0000 last_rx_front 2019-12-07T19:33:41.252305+0000 2019-12-07T19:33:42.488+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252409+0000 last_rx_front 2019-12-07T19:33:41.252456+0000 2019-12-07T19:33:42.488+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.253306+0000 last_rx_front 2019-12-07T19:33:41.252600+0000 2019-12-07T19:33:42.488+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252442+0000 last_rx_front 2019-12-07T19:33:41.252626+0000 2019-12-07T19:33:42.488+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252854+0000 last_rx_front 2019-12-07T19:33:41.253269+0000 2019-12-07T19:33:42.488+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.253337+0000 last_rx_front 2019-12-07T19:33:41.252501+0000 2019-12-07T19:33:42.488+0000 7fe2e2ca2700 20 osd.0 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:42.488+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:42.488+0000 7fe2e2ca2700 20 osd.0 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:42.488+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:42.509+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.559+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 1 osrs, 2 txcs 2019-12-07T19:33:42.559+0000 7fe2dc28f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock osr 0x558e8107a500 2 ios pending 2019-12-07T19:33:42.559+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 6 0x2000~1000 2019-12-07T19:33:42.559+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x2000~1000 crc a2851bee 2019-12-07T19:33:42.559+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 6 0x1b000~1000 2019-12-07T19:33:42.559+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x1b000~1000 crc 1e19645a 2019-12-07T19:33:42.560+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.560+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish osr 0x558e8107a500 2019-12-07T19:33:42.560+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish dequeueing 2019-12-07T19:33:42.610+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.637+0000 7fe2e4535700 10 osd.0 15 tick 2019-12-07T19:33:42.637+0000 7fe2e4535700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:42.637+0000 7fe2e4535700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:42.637+0000 7fe2e4535700 20 osd.0 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:42.660+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.710+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.760+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.810+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.858+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.4 172.21.15.109:0/36938 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:42.858524+0000/6.307750221s send_stamp 6.307750221s delta_ub 18446744073.704209535s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118fc00 con 0x558e8181bc00 2019-12-07T19:33:42.858+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.4 172.21.15.109:0/36938 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:42.858524+0000/6.307750221s send_stamp 6.307750221s delta_ub 18446744073.704209535s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e8182d000 2019-12-07T19:33:42.858+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703727997s,18446744073.704209535s]) 2019-12-07T19:33:42.858+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:42.858524+0000/6.307750221s send_stamp 6.313573840s delta_ub 0.005823619s) v5 -- 0x558e818a9000 con 0x558e8181bc00 2019-12-07T19:33:42.858+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703641926s,18446744073.704209535s]) 2019-12-07T19:33:42.858+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:42.858524+0000/6.307750221s send_stamp 6.313659911s delta_ub 0.005909690s) v5 -- 0x558e818a8a00 con 0x558e8182d000 2019-12-07T19:33:42.860+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.910+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:42.961+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.011+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.061+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.111+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.161+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.211+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.261+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.311+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.361+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.411+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.461+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.511+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.532+0000 7fe2e2ca2700 10 osd.0 15 tick_without_osd_lock 2019-12-07T19:33:43.532+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:43.532+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252746+0000 last_rx_front 2019-12-07T19:33:41.253382+0000 2019-12-07T19:33:43.532+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252531+0000 last_rx_front 2019-12-07T19:33:41.252305+0000 2019-12-07T19:33:43.532+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252409+0000 last_rx_front 2019-12-07T19:33:41.252456+0000 2019-12-07T19:33:43.532+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.253306+0000 last_rx_front 2019-12-07T19:33:41.252600+0000 2019-12-07T19:33:43.532+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252442+0000 last_rx_front 2019-12-07T19:33:41.252626+0000 2019-12-07T19:33:43.532+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252854+0000 last_rx_front 2019-12-07T19:33:41.253269+0000 2019-12-07T19:33:43.533+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.253337+0000 last_rx_front 2019-12-07T19:33:41.252501+0000 2019-12-07T19:33:43.533+0000 7fe2e2ca2700 20 osd.0 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:43.533+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:43.533+0000 7fe2e2ca2700 20 osd.0 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:43.533+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:43.562+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.612+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.662+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.663+0000 7fe2e4535700 10 osd.0 15 tick 2019-12-07T19:33:43.663+0000 7fe2e4535700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:43.663+0000 7fe2e4535700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:43.663+0000 7fe2e4535700 20 osd.0 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:43.712+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.762+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.812+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.862+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.912+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:43.962+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.012+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.041+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.5 172.21.15.109:0/36936 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.041493+0000/7.507816348s send_stamp 7.507816348s delta_ub 0.011894563s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118ea00 con 0x558e8181b800 2019-12-07T19:33:44.041+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011369362s,0.011894563s]) 2019-12-07T19:33:44.041+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.041493+0000/7.507816348s send_stamp 7.496446986s delta_ub 18446744073.698182254s) v5 -- 0x558e818eea00 con 0x558e8181b800 2019-12-07T19:33:44.041+0000 7fe2e7d3c700 10 osd.0 15 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:44.041+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.5 172.21.15.109:0/36936 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.041493+0000/7.507816348s send_stamp 7.507816348s delta_ub 0.011894563s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118ea00 con 0x558e8182c800 2019-12-07T19:33:44.041+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011316427s,0.011894563s]) 2019-12-07T19:33:44.041+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.041493+0000/7.507816348s send_stamp 7.496499921s delta_ub 18446744073.698235189s) v5 -- 0x558e818ee400 con 0x558e8182c800 2019-12-07T19:33:44.062+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.113+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.163+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.213+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.263+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.313+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.363+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.413+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.463+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.513+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.544+0000 7fe2e2ca2700 10 osd.0 15 tick_without_osd_lock 2019-12-07T19:33:44.544+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:44.544+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252746+0000 last_rx_front 2019-12-07T19:33:41.253382+0000 2019-12-07T19:33:44.544+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252531+0000 last_rx_front 2019-12-07T19:33:41.252305+0000 2019-12-07T19:33:44.544+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252409+0000 last_rx_front 2019-12-07T19:33:41.252456+0000 2019-12-07T19:33:44.544+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.253306+0000 last_rx_front 2019-12-07T19:33:41.252600+0000 2019-12-07T19:33:44.544+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252442+0000 last_rx_front 2019-12-07T19:33:41.252626+0000 2019-12-07T19:33:44.544+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.252854+0000 last_rx_front 2019-12-07T19:33:41.253269+0000 2019-12-07T19:33:44.544+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:41.252061+0000 last_rx_back 2019-12-07T19:33:41.253337+0000 last_rx_front 2019-12-07T19:33:41.252501+0000 2019-12-07T19:33:44.544+0000 7fe2e2ca2700 20 osd.0 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:44.544+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:44.544+0000 7fe2e2ca2700 20 osd.0 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:44.544+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:44.563+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.613+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.625+0000 7fe2e4535700 10 osd.0 15 tick 2019-12-07T19:33:44.625+0000 7fe2e4535700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:44.625+0000 7fe2e4535700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:44.625+0000 7fe2e4535700 20 osd.0 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:44.664+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.681+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e818ff400 0x558e818e6c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:44.681+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e818ff400 0x558e818e6c00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:44.681+0000 7fe2e753b700 10 osd.0 15 ms_handle_authentication new session 0x558e818c36c0 con 0x558e818ff400 entity client.admin addr 2019-12-07T19:33:44.681+0000 7fe2e753b700 10 osd.0 15 ms_handle_authentication session 0x558e818c36c0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:44.681+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1043640228 conn(0x558e818ff400 0x558e818e6c00 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=client.4279 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:44.681+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4279 172.21.15.131:0/1043640228 1 ==== command(tid 1: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558e817b4000 con 0x558e818ff400 2019-12-07T19:33:44.681+0000 7fe2d959f700 20 osd.0 15 OSD::ms_dispatch: command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:44.681+0000 7fe2d959f700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:44.681+0000 7fe2d959f700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:44.681+0000 7fe2d959f700 20 osd.0 15 _dispatch 0x558e817b4000 command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:44.682+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/1043640228 -- command_reply(tid 1: 0 ) v1 -- 0x558e8022f340 con 0x558e818ff400 2019-12-07T19:33:44.701+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4279 172.21.15.131:0/1043640228 2 ==== command(tid 2: {"prefix": "flush_pg_stats"}) v1 ==== 52+0+0 (crc 0 0 0) 0x558e817b41e0 con 0x558e818ff400 2019-12-07T19:33:44.701+0000 7fe2d959f700 20 osd.0 15 OSD::ms_dispatch: command(tid 2: {"prefix": "flush_pg_stats"}) v1 2019-12-07T19:33:44.701+0000 7fe2d959f700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:44.701+0000 7fe2d959f700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:44.701+0000 7fe2d959f700 20 osd.0 15 _dispatch 0x558e817b41e0 command(tid 2: {"prefix": "flush_pg_stats"}) v1 2019-12-07T19:33:44.701+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x558e81174700 con 0x558e81748000 2019-12-07T19:33:44.701+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/1043640228 -- command_reply(tid 2: 0 ) v1 -- 0x558e8022f180 con 0x558e818ff400 2019-12-07T19:33:44.702+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1043640228 conn(0x558e818ff400 msgr2=0x558e818e6c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 90 2019-12-07T19:33:44.702+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1043640228 conn(0x558e818ff400 msgr2=0x558e818e6c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:44.702+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1043640228 conn(0x558e818ff400 0x558e818e6c00 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:44.702+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1043640228 conn(0x558e818ff400 0x558e818e6c00 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:44.702+0000 7fe2d959f700 2 osd.0 15 ms_handle_reset con 0x558e818ff400 session 0x558e818c36c0 2019-12-07T19:33:44.714+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 5 osd.0 15 heartbeat osd_stat(store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5,6,7] op hist []) 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 20 osd.0 15 check_full_status cur ratio 1.56826e-06, physical ratio 1.56826e-06, new state none 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 18446744073.708049771s) v5 -- 0x558e818ef600 con 0x558e81749800 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 18446744073.708049771s) v5 -- 0x558e818ef800 con 0x558e81749c00 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 18446744073.692721368s) v5 -- 0x558e818efa00 con 0x558e817ff000 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 18446744073.692721368s) v5 -- 0x558e818efc00 con 0x558e817ff400 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 18446744073.706871233s) v5 -- 0x558e818efe00 con 0x558e817ff800 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 18446744073.706871233s) v5 -- 0x558e811f6000 con 0x558e817ffc00 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 0.005909690s) v5 -- 0x558e811f6200 con 0x558e8181a000 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 0.005909690s) v5 -- 0x558e811f6400 con 0x558e8181a400 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 18446744073.698235189s) v5 -- 0x558e811f6600 con 0x558e817fe000 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 18446744073.698235189s) v5 -- 0x558e811f6800 con 0x558e817fe400 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 0.000490236s) v5 -- 0x558e811f6a00 con 0x558e8181a800 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 0.000490236s) v5 -- 0x558e811f6c00 con 0x558e8181ac00 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 0.019693130s) v5 -- 0x558e818ee800 con 0x558e817fe800 2019-12-07T19:33:44.751+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207132496s delta_ub 0.019693130s) v5 -- 0x558e818eea00 con 0x558e817fec00 2019-12-07T19:33:44.751+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6831/37078 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.209501675s delta_ub 0.002369179s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e81749800 2019-12-07T19:33:44.751+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:41.252746+0000 -> 2019-12-07T19:33:44.752671+0000 last_rx_front 2019-12-07T19:33:41.253382+0000 2019-12-07T19:33:44.751+0000 7fe2e753b700 10 osd.0 15 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:44.751+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002138619s,0.002369179s]) 2019-12-07T19:33:44.752+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6808/37077 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.224312256s delta_ub 0.017179760s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e817ff000 2019-12-07T19:33:44.752+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:41.252531+0000 -> 2019-12-07T19:33:44.752705+0000 last_rx_front 2019-12-07T19:33:41.252305+0000 2019-12-07T19:33:44.752+0000 7fe2e753b700 10 osd.0 15 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:44.752+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016915561s,0.017179760s]) 2019-12-07T19:33:44.752+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6814/37076 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.210136649s delta_ub 0.003004153s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e817ffc00 2019-12-07T19:33:44.752+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:41.252409+0000 last_rx_front 2019-12-07T19:33:41.252456+0000 -> 2019-12-07T19:33:44.752723+0000 2019-12-07T19:33:44.752+0000 7fe2e753b700 10 osd.0 15 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 13 -> 15 (as per caller) 2019-12-07T19:33:44.752+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002721523s,0.003004153s]) 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6827/37078 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.209556888s delta_ub 0.002424392s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e81749c00 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6816/37076 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.210161809s delta_ub 0.003029313s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee200 con 0x558e817ff800 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752671+0000 last_rx_front 2019-12-07T19:33:41.253382+0000 -> 2019-12-07T19:33:44.752733+0000 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:44.752440+0000) and older pending ping(s) 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002132146s,0.002424392s]) 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:41.252409+0000 -> 2019-12-07T19:33:44.752758+0000 last_rx_front 2019-12-07T19:33:44.752723+0000 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:44.752440+0000) and older pending ping(s) 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6806/37077 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.224353127s delta_ub 0.017220631s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e817ff400 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002711369s,0.003029313s]) 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752705+0000 last_rx_front 2019-12-07T19:33:41.252305+0000 -> 2019-12-07T19:33:44.752784+0000 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:44.752440+0000) and older pending ping(s) 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016876732s,0.017220631s]) 2019-12-07T19:33:44.752+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.6 v2:172.21.15.109:6822/36937 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207194526s delta_ub 0.000062030s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e8181a800 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6830/36939 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.188117821s delta_ub 18446744073.690536941s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e817fe800 2019-12-07T19:33:44.752+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:41.252854+0000 -> 2019-12-07T19:33:44.752861+0000 last_rx_front 2019-12-07T19:33:41.253269+0000 2019-12-07T19:33:44.752+0000 7fe2e753b700 10 osd.0 15 maybe_share_map con 0x558e818ffc00 v2:172.21.15.109:6818/36937 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:44.752+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709192724s,0.000062030s]) 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:41.253337+0000 -> 2019-12-07T19:33:44.752873+0000 last_rx_front 2019-12-07T19:33:41.252501+0000 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 10 osd.0 15 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 14 -> 15 (as per caller) 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690104801s,18446744073.690536941s]) 2019-12-07T19:33:44.752+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6804/36936 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.219045112s delta_ub 0.011912616s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e817fe400 2019-12-07T19:33:44.752+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:41.252442+0000 last_rx_front 2019-12-07T19:33:41.252626+0000 -> 2019-12-07T19:33:44.752910+0000 2019-12-07T19:33:44.752+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011443017s,0.011912616s]) 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.6 v2:172.21.15.109:6820/36937 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.207304792s delta_ub 0.000172296s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e8181ac00 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6806/36936 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.219097951s delta_ub 0.011965455s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee200 con 0x558e817fe000 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752861+0000 last_rx_front 2019-12-07T19:33:41.253269+0000 -> 2019-12-07T19:33:44.752947+0000 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got all replies from osd.6 , erase pending ping(sent at 2019-12-07T19:33:44.752440+0000) and older pending ping(s) 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709217690s,0.000172296s]) 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:41.252442+0000 -> 2019-12-07T19:33:44.752963+0000 last_rx_front 2019-12-07T19:33:44.752910+0000 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:44.752440+0000) and older pending ping(s) 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011442768s,0.011965455s]) 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6814/36938 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.201944915s delta_ub 18446744073.704364035s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e8181a000 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:41.253306+0000 -> 2019-12-07T19:33:44.753008+0000 last_rx_front 2019-12-07T19:33:41.252600+0000 2019-12-07T19:33:44.752+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703796247s,18446744073.704364035s]) 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6828/36939 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.188274856s delta_ub 18446744073.690693976s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee200 con 0x558e817fec00 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752873+0000 last_rx_front 2019-12-07T19:33:41.252501+0000 -> 2019-12-07T19:33:44.753028+0000 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:33:44.752440+0000) and older pending ping(s) 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690106884s,18446744073.690693976s]) 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6812/36938 2 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:44.752440+0000/8.207132496s send_stamp 8.202037677s delta_ub 18446744073.704456797s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee200 con 0x558e8181a400 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.753008+0000 last_rx_front 2019-12-07T19:33:41.252600+0000 -> 2019-12-07T19:33:44.753049+0000 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:44.752440+0000) and older pending ping(s) 2019-12-07T19:33:44.752+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703847869s,18446744073.704456797s]) 2019-12-07T19:33:44.764+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.814+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.864+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.914+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:44.964+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.014+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.064+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.114+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.165+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.215+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.265+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.315+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.349+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.3 172.21.15.131:0/37076 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807969424s send_stamp 8.807969424s delta_ub 0.003029313s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee200 con 0x558e8182c400 2019-12-07T19:33:45.349+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002625897s,0.003029313s]) 2019-12-07T19:33:45.349+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807969424s send_stamp 8.805343527s delta_ub 18446744073.706925719s) v5 -- 0x558e818eea00 con 0x558e8182c400 2019-12-07T19:33:45.349+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.3 172.21.15.131:0/37076 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807969424s send_stamp 8.807969424s delta_ub 0.003029313s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e8182d800 2019-12-07T19:33:45.349+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002596897s,0.003029313s]) 2019-12-07T19:33:45.349+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.1 172.21.15.131:0/37078 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807380414s send_stamp 8.807380414s delta_ub 0.002424392s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee200 con 0x558e81749400 2019-12-07T19:33:45.349+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807969424s send_stamp 8.805372527s delta_ub 18446744073.706954719s) v5 -- 0x558e811f6a00 con 0x558e8182d800 2019-12-07T19:33:45.350+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.1 172.21.15.131:0/37078 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807380414s send_stamp 8.807380414s delta_ub 0.002424392s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e8182dc00 2019-12-07T19:33:45.350+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.001977132s,0.002424392s]) 2019-12-07T19:33:45.350+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807380414s send_stamp 8.805403282s delta_ub 18446744073.707574484s) v5 -- 0x558e818ee800 con 0x558e8182dc00 2019-12-07T19:33:45.350+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.001950080s,0.002424392s]) 2019-12-07T19:33:45.350+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.350457+0000/8.807380414s send_stamp 8.805430334s delta_ub 18446744073.707601536s) v5 -- 0x558e818a9800 con 0x558e81749400 2019-12-07T19:33:45.365+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.371+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.7 172.21.15.109:0/36939 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.690693976s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e81748c00 2019-12-07T19:33:45.371+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690068150s,18446744073.690693976s]) 2019-12-07T19:33:45.371+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.826478726s delta_ub 0.019483466s) v5 -- 0x558e818ee800 con 0x558e81748c00 2019-12-07T19:33:45.371+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.7 172.21.15.109:0/36939 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.806995260s delta_ub 18446744073.690693976s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e81749000 2019-12-07T19:33:45.371+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690022920s,18446744073.690693976s]) 2019-12-07T19:33:45.371+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.371556+0000/8.806995260s send_stamp 8.826523956s delta_ub 0.019528696s) v5 -- 0x558e811f6c00 con 0x558e81749000 2019-12-07T19:33:45.415+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.465+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.515+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.516+0000 7fe2e2ca2700 10 osd.0 15 tick_without_osd_lock 2019-12-07T19:33:45.516+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:45.516+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752671+0000 last_rx_front 2019-12-07T19:33:44.752733+0000 2019-12-07T19:33:45.516+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752705+0000 last_rx_front 2019-12-07T19:33:44.752784+0000 2019-12-07T19:33:45.516+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752758+0000 last_rx_front 2019-12-07T19:33:44.752723+0000 2019-12-07T19:33:45.516+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.753008+0000 last_rx_front 2019-12-07T19:33:44.753049+0000 2019-12-07T19:33:45.516+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752963+0000 last_rx_front 2019-12-07T19:33:44.752910+0000 2019-12-07T19:33:45.516+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752861+0000 last_rx_front 2019-12-07T19:33:44.752947+0000 2019-12-07T19:33:45.516+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752873+0000 last_rx_front 2019-12-07T19:33:44.753028+0000 2019-12-07T19:33:45.516+0000 7fe2e2ca2700 20 osd.0 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:45.516+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:45.516+0000 7fe2e2ca2700 20 osd.0 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:45.516+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:45.520+0000 7fe2d5281700 20 osd.0 15 reports for 0 queries 2019-12-07T19:33:45.520+0000 7fe2d5281700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.0 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558e81175180 con 0x558e81748000 2019-12-07T19:33:45.520+0000 7fe2d5281700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x558e81175500 con 0x558e81748000 2019-12-07T19:33:45.565+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:45.565+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.601+0000 7fe2e4535700 10 osd.0 15 tick 2019-12-07T19:33:45.601+0000 7fe2e4535700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:45.601+0000 7fe2e4535700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:45.601+0000 7fe2e4535700 20 osd.0 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:45.615+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.666+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.716+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.766+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.816+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.866+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.916+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:45.953+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.6 172.21.15.109:0/36937 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 0.000172296s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e8182cc00 2019-12-07T19:33:45.953+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709090282s,0.000172296s]) 2019-12-07T19:33:45.953+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408511831s delta_ub 0.000461334s) v5 -- 0x558e811f6a00 con 0x558e8182cc00 2019-12-07T19:33:45.953+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.6 172.21.15.109:0/36937 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408050497s delta_ub 0.000172296s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e802a4c00 2019-12-07T19:33:45.953+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709043461s,0.000172296s]) 2019-12-07T19:33:45.953+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:45.953545+0000/9.408050497s send_stamp 9.408558652s delta_ub 0.000508155s) v5 -- 0x558e811f6800 con 0x558e802a4c00 2019-12-07T19:33:45.966+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.016+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.066+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.116+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.166+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.216+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.267+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.317+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.358+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.4 172.21.15.109:0/36938 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.358819+0000/9.808045243s send_stamp 9.808045243s delta_ub 18446744073.704456797s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e8181bc00 2019-12-07T19:33:46.358+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703778226s,18446744073.704456797s]) 2019-12-07T19:33:46.358+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.358819+0000/9.808045243s send_stamp 9.813818633s delta_ub 0.005773390s) v5 -- 0x558e811f6800 con 0x558e8181bc00 2019-12-07T19:33:46.358+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.4 172.21.15.109:0/36938 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.358819+0000/9.808045243s send_stamp 9.808045243s delta_ub 18446744073.704456797s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e8182d000 2019-12-07T19:33:46.358+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703707459s,18446744073.704456797s]) 2019-12-07T19:33:46.358+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.358819+0000/9.808045243s send_stamp 9.813889400s delta_ub 0.005844157s) v5 -- 0x558e811f6c00 con 0x558e8182d000 2019-12-07T19:33:46.367+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.417+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.450+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.3 172.21.15.131:0/37076 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.450807+0000/9.908320123s send_stamp 9.908320123s delta_ub 0.003121106s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e8182d800 2019-12-07T19:33:46.450+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002729450s,0.003121106s]) 2019-12-07T19:33:46.450+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.3 172.21.15.131:0/37076 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.450807+0000/9.908320123s send_stamp 9.908320123s delta_ub 0.003121106s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8c00 con 0x558e8182c400 2019-12-07T19:33:46.450+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.450807+0000/9.908320123s send_stamp 9.905590673s delta_ub 18446744073.706822166s) v5 -- 0x558e811f6800 con 0x558e8182d800 2019-12-07T19:33:46.450+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002646224s,0.003121106s]) 2019-12-07T19:33:46.450+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.450807+0000/9.908320123s send_stamp 9.905673899s delta_ub 18446744073.706905392s) v5 -- 0x558e818a9800 con 0x558e8182c400 2019-12-07T19:33:46.467+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.473+0000 7fe2e2ca2700 10 osd.0 15 tick_without_osd_lock 2019-12-07T19:33:46.473+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:46.473+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752671+0000 last_rx_front 2019-12-07T19:33:44.752733+0000 2019-12-07T19:33:46.473+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752705+0000 last_rx_front 2019-12-07T19:33:44.752784+0000 2019-12-07T19:33:46.473+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752758+0000 last_rx_front 2019-12-07T19:33:44.752723+0000 2019-12-07T19:33:46.473+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.753008+0000 last_rx_front 2019-12-07T19:33:44.753049+0000 2019-12-07T19:33:46.473+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752963+0000 last_rx_front 2019-12-07T19:33:44.752910+0000 2019-12-07T19:33:46.473+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752861+0000 last_rx_front 2019-12-07T19:33:44.752947+0000 2019-12-07T19:33:46.473+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752873+0000 last_rx_front 2019-12-07T19:33:44.753028+0000 2019-12-07T19:33:46.473+0000 7fe2e2ca2700 20 osd.0 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:46.473+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:46.473+0000 7fe2e2ca2700 20 osd.0 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:46.473+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:46.517+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.535+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.2 172.21.15.131:0/37077 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.536070+0000/10.007789311s send_stamp 10.007789311s delta_ub 0.017220631s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e8182c000 2019-12-07T19:33:46.535+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016924262s,0.017220631s]) 2019-12-07T19:33:46.535+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.536070+0000/10.007789311s send_stamp 9.990865049s delta_ub 18446744073.692627354s) v5 -- 0x558e811f6c00 con 0x558e8182c000 2019-12-07T19:33:46.535+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.2 172.21.15.131:0/37077 2 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:46.536070+0000/10.007789311s send_stamp 10.007789311s delta_ub 0.017220631s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee200 con 0x558e8182d400 2019-12-07T19:33:46.535+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016906994s,0.017220631s]) 2019-12-07T19:33:46.535+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:46.536070+0000/10.007789311s send_stamp 9.990882317s delta_ub 18446744073.692644622s) v5 -- 0x558e818a9800 con 0x558e8182d400 2019-12-07T19:33:46.550+0000 7fe2d859d700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- log(2 entries from seq 1 at 2019-12-07T19:33:37.558565+0000) v1 -- 0x558e817bbb00 con 0x558e80f39000 2019-12-07T19:33:46.565+0000 7fe2e4535700 10 osd.0 15 tick 2019-12-07T19:33:46.565+0000 7fe2e4535700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:46.565+0000 7fe2e4535700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:46.565+0000 7fe2e4535700 20 osd.0 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:46.567+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:33:46.567+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:33:46.567+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:46.567+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:46.567+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:46.567+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:46.567+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:46.567+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:46.567+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:46.567+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:46.567+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:46.567+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:46.567+0000 7fe2dc28f700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:33:46.567+0000 7fe2dc28f700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258772 2019-12-07T19:33:46.567+0000 7fe2dc28f700 5 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.618+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.668+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.718+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.768+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.818+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.868+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.918+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:46.968+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.018+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.069+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.119+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.169+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.219+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.222+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 12 ==== log(last 2) v1 ==== 28+0+0 (crc 0 0 0) 0x558e817b4000 con 0x558e80f39000 2019-12-07T19:33:47.269+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.319+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.369+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.419+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.463+0000 7fe2e2ca2700 10 osd.0 15 tick_without_osd_lock 2019-12-07T19:33:47.463+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:47.463+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752671+0000 last_rx_front 2019-12-07T19:33:44.752733+0000 2019-12-07T19:33:47.463+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752705+0000 last_rx_front 2019-12-07T19:33:44.752784+0000 2019-12-07T19:33:47.463+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752758+0000 last_rx_front 2019-12-07T19:33:44.752723+0000 2019-12-07T19:33:47.463+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.753008+0000 last_rx_front 2019-12-07T19:33:44.753049+0000 2019-12-07T19:33:47.463+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752963+0000 last_rx_front 2019-12-07T19:33:44.752910+0000 2019-12-07T19:33:47.463+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752861+0000 last_rx_front 2019-12-07T19:33:44.752947+0000 2019-12-07T19:33:47.463+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752873+0000 last_rx_front 2019-12-07T19:33:44.753028+0000 2019-12-07T19:33:47.463+0000 7fe2e2ca2700 20 osd.0 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:47.463+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:47.463+0000 7fe2e2ca2700 20 osd.0 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:47.463+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:47.469+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.519+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.569+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.597+0000 7fe2e4535700 10 osd.0 15 tick 2019-12-07T19:33:47.597+0000 7fe2e4535700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:47.597+0000 7fe2e4535700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:47.597+0000 7fe2e4535700 20 osd.0 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:47.620+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.670+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.720+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.770+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.820+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.870+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.920+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:47.970+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.020+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.070+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.120+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.150+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.3 172.21.15.131:0/37076 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.151156+0000/11.608668265s send_stamp 11.608668265s delta_ub 0.002937616s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee200 con 0x558e8182c400 2019-12-07T19:33:48.150+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.3 172.21.15.131:0/37076 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.151156+0000/11.608668265s send_stamp 11.608668265s delta_ub 0.002937616s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e8182d800 2019-12-07T19:33:48.150+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002678611s,0.002937616s]) 2019-12-07T19:33:48.150+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.151156+0000/11.608668265s send_stamp 11.605989654s delta_ub 18446744073.706873005s) v5 -- 0x558e818a9800 con 0x558e8182c400 2019-12-07T19:33:48.150+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002654182s,0.002937616s]) 2019-12-07T19:33:48.150+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.151156+0000/11.608668265s send_stamp 11.606014083s delta_ub 18446744073.706897434s) v5 -- 0x558e811f6800 con 0x558e8182d800 2019-12-07T19:33:48.171+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.221+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.271+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.321+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.371+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.421+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.471+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.504+0000 7fe2e2ca2700 10 osd.0 15 tick_without_osd_lock 2019-12-07T19:33:48.504+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:48.504+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752671+0000 last_rx_front 2019-12-07T19:33:44.752733+0000 2019-12-07T19:33:48.504+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752705+0000 last_rx_front 2019-12-07T19:33:44.752784+0000 2019-12-07T19:33:48.504+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752758+0000 last_rx_front 2019-12-07T19:33:44.752723+0000 2019-12-07T19:33:48.504+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.753008+0000 last_rx_front 2019-12-07T19:33:44.753049+0000 2019-12-07T19:33:48.504+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752963+0000 last_rx_front 2019-12-07T19:33:44.752910+0000 2019-12-07T19:33:48.504+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752861+0000 last_rx_front 2019-12-07T19:33:44.752947+0000 2019-12-07T19:33:48.504+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:44.752440+0000 last_rx_back 2019-12-07T19:33:44.752873+0000 last_rx_front 2019-12-07T19:33:44.753028+0000 2019-12-07T19:33:48.504+0000 7fe2e2ca2700 20 osd.0 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:48.504+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:48.504+0000 7fe2e2ca2700 20 osd.0 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:48.504+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:48.521+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.571+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:48.571+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.591+0000 7fe2e4535700 10 osd.0 15 tick 2019-12-07T19:33:48.591+0000 7fe2e4535700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:48.591+0000 7fe2e4535700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:48.591+0000 7fe2e4535700 20 osd.0 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:48.621+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.671+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.722+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.772+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.822+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.851+0000 7fe2c3a5e700 5 osd.0 15 heartbeat osd_stat(store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5,6,7] op hist []) 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 20 osd.0 15 check_full_status cur ratio 1.56826e-06, physical ratio 1.56826e-06, new state none 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 18446744073.707601536s) v5 -- 0x558e818ee400 con 0x558e81749800 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 18446744073.707601536s) v5 -- 0x558e818efe00 con 0x558e81749c00 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 18446744073.692644622s) v5 -- 0x558e818eee00 con 0x558e817ff000 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 18446744073.692644622s) v5 -- 0x558e818eec00 con 0x558e817ff400 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 18446744073.706897434s) v5 -- 0x558e818ef000 con 0x558e817ff800 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 18446744073.706897434s) v5 -- 0x558e818ee600 con 0x558e817ffc00 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 0.005844157s) v5 -- 0x558e811f6400 con 0x558e8181a000 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 0.005844157s) v5 -- 0x558e818ef400 con 0x558e8181a400 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 18446744073.698108848s) v5 -- 0x558e818ef800 con 0x558e817fe000 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 18446744073.698108848s) v5 -- 0x558e818efc00 con 0x558e817fe400 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 0.000508155s) v5 -- 0x558e818ef600 con 0x558e8181a800 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 0.000508155s) v5 -- 0x558e818efa00 con 0x558e8181ac00 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 0.019528696s) v5 -- 0x558e818a9600 con 0x558e817fe800 2019-12-07T19:33:48.852+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307421034s delta_ub 0.019528696s) v5 -- 0x558e818a9c00 con 0x558e817fec00 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6827/37078 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.309767486s delta_ub 0.002346452s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e81749c00 2019-12-07T19:33:48.852+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6814/37076 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.310393460s delta_ub 0.002972426s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e817ffc00 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:44.752671+0000 last_rx_front 2019-12-07T19:33:44.752733+0000 -> 2019-12-07T19:33:48.852951+0000 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002124089s,0.002346452s]) 2019-12-07T19:33:48.852+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:44.752758+0000 last_rx_front 2019-12-07T19:33:44.752723+0000 -> 2019-12-07T19:33:48.852963+0000 2019-12-07T19:33:48.852+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002738081s,0.002972426s]) 2019-12-07T19:33:48.852+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6816/37076 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.310418950s delta_ub 0.002997916s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818eea00 con 0x558e817ff800 2019-12-07T19:33:48.852+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:44.752758+0000 -> 2019-12-07T19:33:48.852979+0000 last_rx_front 2019-12-07T19:33:48.852963+0000 2019-12-07T19:33:48.852+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:48.852729+0000) and older pending ping(s) 2019-12-07T19:33:48.852+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6831/37078 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.309797583s delta_ub 0.002376549s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e81749800 2019-12-07T19:33:48.852+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002748004s,0.002997916s]) 2019-12-07T19:33:48.852+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:44.752671+0000 -> 2019-12-07T19:33:48.853004+0000 last_rx_front 2019-12-07T19:33:48.852951+0000 2019-12-07T19:33:48.852+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:48.852729+0000) and older pending ping(s) 2019-12-07T19:33:48.852+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002101184s,0.002376549s]) 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6806/37077 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.324682403s delta_ub 0.017261369s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e817ff400 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:44.752705+0000 last_rx_front 2019-12-07T19:33:44.752784+0000 -> 2019-12-07T19:33:48.853025+0000 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016965556s,0.017261369s]) 2019-12-07T19:33:48.852+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6808/37077 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.324641793s delta_ub 0.017220759s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e817ff000 2019-12-07T19:33:48.852+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:44.752705+0000 -> 2019-12-07T19:33:48.853039+0000 last_rx_front 2019-12-07T19:33:48.853025+0000 2019-12-07T19:33:48.852+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:48.852729+0000) and older pending ping(s) 2019-12-07T19:33:48.852+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016910445s,0.017220759s]) 2019-12-07T19:33:48.852+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.6 v2:172.21.15.109:6822/36937 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307478930s delta_ub 0.000057896s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e8181a800 2019-12-07T19:33:48.852+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6806/36936 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.319293876s delta_ub 0.011872842s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818eea00 con 0x558e817fe000 2019-12-07T19:33:48.852+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:44.752861+0000 -> 2019-12-07T19:33:48.853173+0000 last_rx_front 2019-12-07T19:33:44.752947+0000 2019-12-07T19:33:48.852+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709165804s,0.000057896s]) 2019-12-07T19:33:48.852+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:44.752963+0000 -> 2019-12-07T19:33:48.853183+0000 last_rx_front 2019-12-07T19:33:44.752910+0000 2019-12-07T19:33:48.852+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011419039s,0.011872842s]) 2019-12-07T19:33:48.852+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6828/36939 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.288469078s delta_ub 18446744073.690599660s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818eea00 con 0x558e817fec00 2019-12-07T19:33:48.852+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:44.752873+0000 last_rx_front 2019-12-07T19:33:44.753028+0000 -> 2019-12-07T19:33:48.853215+0000 2019-12-07T19:33:48.852+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690113776s,18446744073.690599660s]) 2019-12-07T19:33:48.852+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6804/36936 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.319354302s delta_ub 0.011933268s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e817fe400 2019-12-07T19:33:48.852+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853183+0000 last_rx_front 2019-12-07T19:33:44.752910+0000 -> 2019-12-07T19:33:48.853235+0000 2019-12-07T19:33:48.852+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:48.852729+0000) and older pending ping(s) 2019-12-07T19:33:48.852+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011427037s,0.011933268s]) 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6814/36938 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.302211605s delta_ub 18446744073.704342187s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e8181a000 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:44.753008+0000 -> 2019-12-07T19:33:48.853251+0000 last_rx_front 2019-12-07T19:33:44.753049+0000 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703820515s,18446744073.704342187s]) 2019-12-07T19:33:48.852+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6812/36938 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.302308958s delta_ub 18446744073.704439540s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818eea00 con 0x558e8181a400 2019-12-07T19:33:48.852+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853251+0000 last_rx_front 2019-12-07T19:33:44.753049+0000 -> 2019-12-07T19:33:48.853304+0000 2019-12-07T19:33:48.852+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:48.852729+0000) and older pending ping(s) 2019-12-07T19:33:48.852+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703864534s,18446744073.704439540s]) 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.6 v2:172.21.15.109:6820/36937 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.307635018s delta_ub 0.000213984s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e8181ac00 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853173+0000 last_rx_front 2019-12-07T19:33:44.752947+0000 -> 2019-12-07T19:33:48.853315+0000 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got all replies from osd.6 , erase pending ping(sent at 2019-12-07T19:33:48.852729+0000) and older pending ping(s) 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709179092s,0.000213984s]) 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6830/36939 3 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.852729+0000/12.307421034s send_stamp 12.288612502s delta_ub 18446744073.690743084s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e817fe800 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:44.752873+0000 -> 2019-12-07T19:33:48.853338+0000 last_rx_front 2019-12-07T19:33:48.853215+0000 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:33:48.852729+0000) and older pending ping(s) 2019-12-07T19:33:48.852+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690134354s,18446744073.690743084s]) 2019-12-07T19:33:48.871+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.7 172.21.15.109:0/36939 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.690743084s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e81749000 2019-12-07T19:33:48.871+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690029510s,18446744073.690743084s]) 2019-12-07T19:33:48.871+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.326851678s delta_ub 0.019522106s) v5 -- 0x558e818a9600 con 0x558e81749000 2019-12-07T19:33:48.871+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.7 172.21.15.109:0/36939 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.307329572s delta_ub 18446744073.690743084s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e81748c00 2019-12-07T19:33:48.871+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.689986662s,18446744073.690743084s]) 2019-12-07T19:33:48.871+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:48.871890+0000/12.307329572s send_stamp 12.326894526s delta_ub 0.019564954s) v5 -- 0x558e818efa00 con 0x558e81748c00 2019-12-07T19:33:48.872+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.922+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:48.972+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.022+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.072+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.122+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.172+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.222+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.258+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.4 172.21.15.109:0/36938 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.259135+0000/12.708361738s send_stamp 12.708361738s delta_ub 18446744073.704439540s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e8181bc00 2019-12-07T19:33:49.258+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703791488s,18446744073.704439540s]) 2019-12-07T19:33:49.258+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.259135+0000/12.708361738s send_stamp 12.714121866s delta_ub 0.005760128s) v5 -- 0x558e818ef600 con 0x558e8181bc00 2019-12-07T19:33:49.258+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.4 172.21.15.109:0/36938 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.259135+0000/12.708361738s send_stamp 12.708361738s delta_ub 18446744073.704439540s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e8182d000 2019-12-07T19:33:49.258+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703743664s,18446744073.704439540s]) 2019-12-07T19:33:49.258+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.259135+0000/12.708361738s send_stamp 12.714169690s delta_ub 0.005807952s) v5 -- 0x558e818efa00 con 0x558e8182d000 2019-12-07T19:33:49.273+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.323+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.373+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.423+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.473+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.475+0000 7fe2e2ca2700 10 osd.0 15 tick_without_osd_lock 2019-12-07T19:33:49.475+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:49.475+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853004+0000 last_rx_front 2019-12-07T19:33:48.852951+0000 2019-12-07T19:33:49.475+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853039+0000 last_rx_front 2019-12-07T19:33:48.853025+0000 2019-12-07T19:33:49.475+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.852979+0000 last_rx_front 2019-12-07T19:33:48.852963+0000 2019-12-07T19:33:49.475+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853251+0000 last_rx_front 2019-12-07T19:33:48.853304+0000 2019-12-07T19:33:49.475+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853183+0000 last_rx_front 2019-12-07T19:33:48.853235+0000 2019-12-07T19:33:49.475+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853173+0000 last_rx_front 2019-12-07T19:33:48.853315+0000 2019-12-07T19:33:49.475+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853338+0000 last_rx_front 2019-12-07T19:33:48.853215+0000 2019-12-07T19:33:49.475+0000 7fe2e2ca2700 20 osd.0 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:49.475+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:49.475+0000 7fe2e2ca2700 20 osd.0 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:49.475+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:49.523+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.573+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.612+0000 7fe2e4535700 10 osd.0 15 tick 2019-12-07T19:33:49.612+0000 7fe2e4535700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:49.612+0000 7fe2e4535700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:49.612+0000 7fe2e4535700 20 osd.0 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:49.623+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.673+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.723+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.773+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.823+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.850+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.3 172.21.15.131:0/37076 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.851502+0000/13.309014358s send_stamp 13.309014358s delta_ub 0.002997916s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e8182d800 2019-12-07T19:33:49.850+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002685038s,0.002997916s]) 2019-12-07T19:33:49.850+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.851502+0000/13.309014358s send_stamp 13.306329320s delta_ub 18446744073.706866578s) v5 -- 0x558e818ef600 con 0x558e8182d800 2019-12-07T19:33:49.850+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.3 172.21.15.131:0/37076 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.851502+0000/13.309014358s send_stamp 13.309014358s delta_ub 0.002997916s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818eea00 con 0x558e8182c400 2019-12-07T19:33:49.851+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002559256s,0.002997916s]) 2019-12-07T19:33:49.851+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.851502+0000/13.309014358s send_stamp 13.306455102s delta_ub 18446744073.706992360s) v5 -- 0x558e818a9c00 con 0x558e8182c400 2019-12-07T19:33:49.874+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.924+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:49.941+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.5 172.21.15.109:0/36936 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.941865+0000/13.408188910s send_stamp 13.408188910s delta_ub 0.011933268s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9800 con 0x558e8182c800 2019-12-07T19:33:49.941+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011333146s,0.011933268s]) 2019-12-07T19:33:49.941+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.941865+0000/13.408188910s send_stamp 13.396855764s delta_ub 18446744073.698218470s) v5 -- 0x558e818a9c00 con 0x558e8182c800 2019-12-07T19:33:49.941+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.5 172.21.15.109:0/36936 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:49.941865+0000/13.408188910s send_stamp 13.408188910s delta_ub 0.011933268s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818eea00 con 0x558e8181b800 2019-12-07T19:33:49.941+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011282213s,0.011933268s]) 2019-12-07T19:33:49.941+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:49.941865+0000/13.408188910s send_stamp 13.396906697s delta_ub 18446744073.698269403s) v5 -- 0x558e818a9c00 con 0x558e8181b800 2019-12-07T19:33:49.974+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.024+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.074+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.124+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.174+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.224+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.274+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.324+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.374+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.425+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.475+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.505+0000 7fe2e2ca2700 10 osd.0 15 tick_without_osd_lock 2019-12-07T19:33:50.505+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:50.505+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853004+0000 last_rx_front 2019-12-07T19:33:48.852951+0000 2019-12-07T19:33:50.505+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853039+0000 last_rx_front 2019-12-07T19:33:48.853025+0000 2019-12-07T19:33:50.505+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.852979+0000 last_rx_front 2019-12-07T19:33:48.852963+0000 2019-12-07T19:33:50.505+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853251+0000 last_rx_front 2019-12-07T19:33:48.853304+0000 2019-12-07T19:33:50.505+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853183+0000 last_rx_front 2019-12-07T19:33:48.853235+0000 2019-12-07T19:33:50.505+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853173+0000 last_rx_front 2019-12-07T19:33:48.853315+0000 2019-12-07T19:33:50.505+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:48.852729+0000 last_rx_back 2019-12-07T19:33:48.853338+0000 last_rx_front 2019-12-07T19:33:48.853215+0000 2019-12-07T19:33:50.505+0000 7fe2e2ca2700 20 osd.0 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:50.505+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:50.505+0000 7fe2e2ca2700 20 osd.0 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:50.505+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:50.520+0000 7fe2d5281700 20 osd.0 15 reports for 0 queries 2019-12-07T19:33:50.520+0000 7fe2d5281700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.0 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558e81175880 con 0x558e81748000 2019-12-07T19:33:50.520+0000 7fe2d5281700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x558e81174700 con 0x558e81748000 2019-12-07T19:33:50.525+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.575+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.625+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.650+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.1 172.21.15.131:0/37078 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:50.650794+0000/14.107717608s send_stamp 14.107717608s delta_ub 0.002376549s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8e00 con 0x558e8182dc00 2019-12-07T19:33:50.650+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002103291s,0.002376549s]) 2019-12-07T19:33:50.650+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:50.650794+0000/14.107717608s send_stamp 14.105614317s delta_ub 18446744073.707448325s) v5 -- 0x558e818efa00 con 0x558e8182dc00 2019-12-07T19:33:50.650+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.1 172.21.15.131:0/37078 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:50.650794+0000/14.107717608s send_stamp 14.107717608s delta_ub 0.002376549s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9800 con 0x558e81749400 2019-12-07T19:33:50.650+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002076559s,0.002376549s]) 2019-12-07T19:33:50.650+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:50.650794+0000/14.107717608s send_stamp 14.105641049s delta_ub 18446744073.707475057s) v5 -- 0x558e818a9c00 con 0x558e81749400 2019-12-07T19:33:50.661+0000 7fe2e4535700 10 osd.0 15 tick 2019-12-07T19:33:50.661+0000 7fe2e4535700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:50.661+0000 7fe2e4535700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:50.661+0000 7fe2e4535700 20 osd.0 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:50.675+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.725+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.775+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.825+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.875+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.926+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:50.976+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.026+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.076+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.126+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 5 osd.0 15 heartbeat osd_stat(store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5,6,7] op hist []) 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 20 osd.0 15 check_full_status cur ratio 1.56826e-06, physical ratio 1.56826e-06, new state none 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 18446744073.707475057s) v5 -- 0x558e818a8a00 con 0x558e81749800 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 18446744073.707475057s) v5 -- 0x558e811f6600 con 0x558e81749c00 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 18446744073.692641171s) v5 -- 0x558e811f6200 con 0x558e817ff000 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 18446744073.692641171s) v5 -- 0x558e811f6e00 con 0x558e817ff400 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 18446744073.706992360s) v5 -- 0x558e811f7000 con 0x558e817ff800 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 18446744073.706992360s) v5 -- 0x558e811f7200 con 0x558e817ffc00 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 0.005807952s) v5 -- 0x558e811f7400 con 0x558e8181a000 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 0.005807952s) v5 -- 0x558e811f7600 con 0x558e8181a400 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 18446744073.698269403s) v5 -- 0x558e811f7800 con 0x558e817fe000 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 18446744073.698269403s) v5 -- 0x558e811f7a00 con 0x558e817fe400 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 0.000372524s) v5 -- 0x558e811f7c00 con 0x558e8181a800 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 0.000372524s) v5 -- 0x558e811f6000 con 0x558e8181ac00 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 0.019564954s) v5 -- 0x558e811f6a00 con 0x558e817fe800 2019-12-07T19:33:51.152+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607692400s delta_ub 0.019564954s) v5 -- 0x558e811f6800 con 0x558e817fec00 2019-12-07T19:33:51.152+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6816/37076 4 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.610695690s delta_ub 0.003003290s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818eea00 con 0x558e817ff800 2019-12-07T19:33:51.152+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:48.852979+0000 -> 2019-12-07T19:33:51.153242+0000 last_rx_front 2019-12-07T19:33:48.852963+0000 2019-12-07T19:33:51.152+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002761564s,0.003003290s]) 2019-12-07T19:33:51.152+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6831/37078 4 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.610080106s delta_ub 0.002387706s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e81749800 2019-12-07T19:33:51.152+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:48.853004+0000 -> 2019-12-07T19:33:51.153258+0000 last_rx_front 2019-12-07T19:33:48.852951+0000 2019-12-07T19:33:51.152+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002130099s,0.002387706s]) 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 0 -- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e81749c00 msgr2=0x558e817ef080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2019-12-07T19:33:51.152+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6814/37076 4 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.610732655s delta_ub 0.003040255s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e817ffc00 2019-12-07T19:33:51.152+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153242+0000 last_rx_front 2019-12-07T19:33:48.852963+0000 -> 2019-12-07T19:33:51.153313+0000 2019-12-07T19:33:51.152+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:51.153000+0000) and older pending ping(s) 2019-12-07T19:33:51.152+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002727691s,0.003040255s]) 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6827/37078 4 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.610103417s delta_ub 0.002411017s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e81749c00 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153258+0000 last_rx_front 2019-12-07T19:33:48.852951+0000 -> 2019-12-07T19:33:51.153336+0000 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:51.153000+0000) and older pending ping(s) 2019-12-07T19:33:51.152+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6808/37077 4 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.624910352s delta_ub 0.017217952s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e817ff000 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002075559s,0.002411017s]) 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e81749c00 msgr2=0x558e817ef080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 73 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e81749c00 msgr2=0x558e817ef080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:51.152+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6806/36936 4 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.619562016s delta_ub 0.011869616s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818eea00 con 0x558e817fe000 2019-12-07T19:33:51.152+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:48.853039+0000 -> 2019-12-07T19:33:51.153377+0000 last_rx_front 2019-12-07T19:33:48.853025+0000 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e81749c00 0x558e817ef080 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:51.152+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016840770s,0.017217952s]) 2019-12-07T19:33:51.152+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:48.853183+0000 -> 2019-12-07T19:33:51.153392+0000 last_rx_front 2019-12-07T19:33:48.853235+0000 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e81749c00 0x558e817ef080 crc :-1 s=READY pgs=7 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:51.152+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011477566s,0.011869616s]) 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6806/37077 4 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.624957410s delta_ub 0.017265010s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e817ff400 2019-12-07T19:33:51.152+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6828/36939 4 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.588694208s delta_ub 18446744073.690553424s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818eea00 con 0x558e817fec00 2019-12-07T19:33:51.152+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6804/36936 4 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.619614740s delta_ub 0.011922340s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e817fe400 2019-12-07T19:33:51.152+0000 7fe2d3a7e700 20 osd.0 15 heartbeat_reset con 0x558e81749c00 s 0x558e80245d40 2019-12-07T19:33:51.152+0000 7fe2d3a7e700 10 osd.0 15 heartbeat_reset failed hb con 0x558e81749c00 for osd.1, reopening 2019-12-07T19:33:51.152+0000 7fe2d3a7e700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558e81749800 msgr2=0x558e817eeb00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2019-12-07T19:33:51.152+0000 7fe2d3a7e700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558e81749800 0x558e817eeb00 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:51.152+0000 7fe2d3a7e700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558e818d7800 0x558e81847700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:51.152+0000 7fe2d3a7e700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e818d7400 0x558e818e4b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153377+0000 last_rx_front 2019-12-07T19:33:48.853025+0000 -> 2019-12-07T19:33:51.153481+0000 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:51.153000+0000) and older pending ping(s) 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016784704s,0.017265010s]) 2019-12-07T19:33:51.152+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:48.853338+0000 last_rx_front 2019-12-07T19:33:48.853215+0000 -> 2019-12-07T19:33:51.153496+0000 2019-12-07T19:33:51.152+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690057841s,18446744073.690553424s]) 2019-12-07T19:33:51.152+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153392+0000 last_rx_front 2019-12-07T19:33:48.853235+0000 -> 2019-12-07T19:33:51.153509+0000 2019-12-07T19:33:51.152+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:51.153000+0000) and older pending ping(s) 2019-12-07T19:33:51.152+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011413227s,0.011922340s]) 2019-12-07T19:33:51.152+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6812/36938 4 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.602516423s delta_ub 18446744073.704375639s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818eea00 con 0x558e8181a400 2019-12-07T19:33:51.152+0000 7fe2e7d3c700 25 osd.0 15 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:48.853251+0000 last_rx_front 2019-12-07T19:33:48.853304+0000 -> 2019-12-07T19:33:51.153527+0000 2019-12-07T19:33:51.152+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703848758s,18446744073.704375639s]) 2019-12-07T19:33:51.152+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.6 v2:172.21.15.109:6822/36937 4 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.607818995s delta_ub 0.000126595s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e8181a800 2019-12-07T19:33:51.152+0000 7fe2e753b700 25 osd.0 15 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:48.853173+0000 -> 2019-12-07T19:33:51.153611+0000 last_rx_front 2019-12-07T19:33:48.853315+0000 2019-12-07T19:33:51.152+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709067909s,0.000126595s]) 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6814/36938 4 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.602486955s delta_ub 18446744073.704346171s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef800 con 0x558e8181a000 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:48.853251+0000 -> 2019-12-07T19:33:51.153637+0000 last_rx_front 2019-12-07T19:33:51.153527+0000 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:51.153000+0000) and older pending ping(s) 2019-12-07T19:33:51.152+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558e818d7800 0x558e81847700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703709618s,18446744073.704346171s]) 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6830/36939 4 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.588938414s delta_ub 18446744073.690797630s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef800 con 0x558e817fe800 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:48.853338+0000 -> 2019-12-07T19:33:51.153670+0000 last_rx_front 2019-12-07T19:33:51.153496+0000 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:33:51.153000+0000) and older pending ping(s) 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690128228s,18446744073.690797630s]) 2019-12-07T19:33:51.152+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.6 v2:172.21.15.109:6820/36937 4 ==== osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.153000+0000/14.607692400s send_stamp 14.608000474s delta_ub 0.000308074s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef800 con 0x558e8181ac00 2019-12-07T19:33:51.153+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153611+0000 last_rx_front 2019-12-07T19:33:48.853315+0000 -> 2019-12-07T19:33:51.153694+0000 2019-12-07T19:33:51.153+0000 7fe2e6d3a700 25 osd.0 15 handle_osd_ping got all replies from osd.6 , erase pending ping(sent at 2019-12-07T19:33:51.153000+0000) and older pending ping(s) 2019-12-07T19:33:51.153+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709166494s,0.000308074s]) 2019-12-07T19:33:51.153+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e818d7400 0x558e818e4b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:51.153+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] conn(0x558e818d7800 0x558e81847700 crc :-1 s=READY pgs=9 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:51.153+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e818d7400 msgr2=0x558e818e4b00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 73 2019-12-07T19:33:51.153+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e818d7400 msgr2=0x558e818e4b00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:51.153+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e818d7400 0x558e818e4b00 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:51.153+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e818d7400 0x558e818e4b00 crc :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.200000 2019-12-07T19:33:51.176+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.226+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.235+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.2 172.21.15.131:0/37077 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.236372+0000/14.708091194s send_stamp 14.708091194s delta_ub 0.017265010s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818eea00 con 0x558e8182d400 2019-12-07T19:33:51.235+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.2 172.21.15.131:0/37077 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.236372+0000/14.708091194s send_stamp 14.708091194s delta_ub 0.017265010s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef200 con 0x558e8182c000 2019-12-07T19:33:51.235+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016902372s,0.017265010s]) 2019-12-07T19:33:51.235+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.236372+0000/14.708091194s send_stamp 14.691188822s delta_ub 18446744073.692649244s) v5 -- 0x558e811f6800 con 0x558e8182d400 2019-12-07T19:33:51.235+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016877195s,0.017265010s]) 2019-12-07T19:33:51.235+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.236372+0000/14.708091194s send_stamp 14.691213999s delta_ub 18446744073.692674421s) v5 -- 0x558e811f7400 con 0x558e8182c000 2019-12-07T19:33:51.276+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.326+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.354+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e818d7400 0x558e818e4b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:51.354+0000 7fe2e6d3a700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] conn(0x558e818d7400 0x558e818e4b00 crc :-1 s=READY pgs=9 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:51.376+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.426+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.477+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.505+0000 7fe2e2ca2700 10 osd.0 15 tick_without_osd_lock 2019-12-07T19:33:51.505+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:51.505+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153258+0000 last_rx_front 2019-12-07T19:33:51.153336+0000 2019-12-07T19:33:51.505+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153377+0000 last_rx_front 2019-12-07T19:33:51.153481+0000 2019-12-07T19:33:51.505+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153242+0000 last_rx_front 2019-12-07T19:33:51.153313+0000 2019-12-07T19:33:51.505+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153637+0000 last_rx_front 2019-12-07T19:33:51.153527+0000 2019-12-07T19:33:51.505+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153392+0000 last_rx_front 2019-12-07T19:33:51.153509+0000 2019-12-07T19:33:51.505+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153611+0000 last_rx_front 2019-12-07T19:33:51.153694+0000 2019-12-07T19:33:51.505+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153670+0000 last_rx_front 2019-12-07T19:33:51.153496+0000 2019-12-07T19:33:51.505+0000 7fe2e2ca2700 20 osd.0 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:51.505+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:51.505+0000 7fe2e2ca2700 20 osd.0 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:51.505+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:51.527+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.577+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:33:51.577+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:33:51.577+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:51.577+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:51.577+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:51.577+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:51.577+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:51.577+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:51.577+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:51.577+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:51.577+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:51.577+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:51.577+0000 7fe2dc28f700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:33:51.577+0000 7fe2dc28f700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258772 2019-12-07T19:33:51.577+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 0 osrs, 0 txcs 2019-12-07T19:33:51.577+0000 7fe2dc28f700 5 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.627+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.654+0000 7fe2e4535700 10 osd.0 15 tick 2019-12-07T19:33:51.654+0000 7fe2e4535700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:51.654+0000 7fe2e4535700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:51.654+0000 7fe2e4535700 20 osd.0 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:51.677+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.727+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.777+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.827+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.853+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.6 172.21.15.109:0/36937 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 0.000308074s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e8182cc00 2019-12-07T19:33:51.853+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709085251s,0.000308074s]) 2019-12-07T19:33:51.853+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308854641s delta_ub 0.000466365s) v5 -- 0x558e811f7c00 con 0x558e8182cc00 2019-12-07T19:33:51.853+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.6 172.21.15.109:0/36937 3 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308388276s delta_ub 0.000308074s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee000 con 0x558e802a4c00 2019-12-07T19:33:51.853+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709040352s,0.000308074s]) 2019-12-07T19:33:51.853+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:51.853882+0000/15.308388276s send_stamp 15.308899540s delta_ub 0.000511264s) v5 -- 0x558e811f7a00 con 0x558e802a4c00 2019-12-07T19:33:51.877+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.927+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:51.977+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.028+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.078+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.128+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.178+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.228+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.278+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.328+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.336+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.2 172.21.15.131:0/37077 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.336720+0000/15.808438677s send_stamp 15.808438677s delta_ub 0.017125179s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ef800 con 0x558e8182c000 2019-12-07T19:33:52.336+0000 7fe2e6d3a700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016878490s,0.017125179s]) 2019-12-07T19:33:52.336+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.336720+0000/15.808438677s send_stamp 15.791560187s delta_ub 18446744073.692673126s) v5 -- 0x558e811f7400 con 0x558e8182c000 2019-12-07T19:33:52.336+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.2 172.21.15.131:0/37077 4 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.336720+0000/15.808438677s send_stamp 15.808438677s delta_ub 0.017125179s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e811f7600 con 0x558e8182d400 2019-12-07T19:33:52.336+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016831893s,0.017125179s]) 2019-12-07T19:33:52.336+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.336720+0000/15.808438677s send_stamp 15.791606784s delta_ub 18446744073.692719723s) v5 -- 0x558e811f6800 con 0x558e8182d400 2019-12-07T19:33:52.378+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.429+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.479+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.520+0000 7fe2e2ca2700 10 osd.0 15 tick_without_osd_lock 2019-12-07T19:33:52.520+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffdb000/0x40000000/0x1680000000, data 0x9886/0x23000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:52.520+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153258+0000 last_rx_front 2019-12-07T19:33:51.153336+0000 2019-12-07T19:33:52.520+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153377+0000 last_rx_front 2019-12-07T19:33:51.153481+0000 2019-12-07T19:33:52.520+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153242+0000 last_rx_front 2019-12-07T19:33:51.153313+0000 2019-12-07T19:33:52.520+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153637+0000 last_rx_front 2019-12-07T19:33:51.153527+0000 2019-12-07T19:33:52.520+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153392+0000 last_rx_front 2019-12-07T19:33:51.153509+0000 2019-12-07T19:33:52.520+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153611+0000 last_rx_front 2019-12-07T19:33:51.153694+0000 2019-12-07T19:33:52.520+0000 7fe2e2ca2700 25 osd.0 15 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153670+0000 last_rx_front 2019-12-07T19:33:51.153496+0000 2019-12-07T19:33:52.520+0000 7fe2e2ca2700 20 osd.0 15 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:52.520+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:52.520+0000 7fe2e2ca2700 20 osd.0 15 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:52.520+0000 7fe2e2ca2700 10 osd.0 15 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:52.529+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.579+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.619+0000 7fe2e4535700 10 osd.0 15 tick 2019-12-07T19:33:52.619+0000 7fe2e4535700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:52.619+0000 7fe2e4535700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:52.619+0000 7fe2e4535700 20 osd.0 15 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:52.629+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.679+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.729+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.751+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.3 172.21.15.131:0/37076 6 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.751844+0000/16.209356665s send_stamp 16.209356665s delta_ub 0.003040255s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e8182d800 2019-12-07T19:33:52.751+0000 7fe2e753b700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002697511s,0.003040255s]) 2019-12-07T19:33:52.751+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.751844+0000/16.209356665s send_stamp 16.206659154s delta_ub 18446744073.706854105s) v5 -- 0x558e811f7a00 con 0x558e8182d800 2019-12-07T19:33:52.751+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.3 172.21.15.131:0/37076 6 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.751844+0000/16.209356665s send_stamp 16.209356665s delta_ub 0.003040255s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818eea00 con 0x558e8182c400 2019-12-07T19:33:52.751+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002677075s,0.003040255s]) 2019-12-07T19:33:52.751+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.751844+0000/16.209356665s send_stamp 16.206679590s delta_ub 18446744073.706874541s) v5 -- 0x558e811f6800 con 0x558e8182c400 2019-12-07T19:33:52.751+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.131:0/37076 conn(0x558e8182d800 msgr2=0x558e81845080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 82 2019-12-07T19:33:52.751+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.131:0/37076 conn(0x558e8182d800 msgr2=0x558e81845080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:52.751+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.131:0/37076 conn(0x558e8182d800 0x558e81845080 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:52.751+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.131:0/37076 conn(0x558e8182d800 0x558e81845080 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:52.751+0000 7fe2d1a7a700 20 osd.0 15 heartbeat_reset con 0x558e8182d800 s 0x558e818c2340 2019-12-07T19:33:52.751+0000 7fe2d1a7a700 10 osd.0 15 heartbeat_reset closing (old) failed hb con 0x558e8182d800 2019-12-07T19:33:52.751+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.131:0/37076 conn(0x558e8182c400 msgr2=0x558e81819180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 68 2019-12-07T19:33:52.751+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.131:0/37076 conn(0x558e8182c400 msgr2=0x558e81819180 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:52.751+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.131:0/37076 conn(0x558e8182c400 0x558e81819180 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:52.751+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.131:0/37076 conn(0x558e8182c400 0x558e81819180 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:52.751+0000 7fe2d0a78700 20 osd.0 15 heartbeat_reset con 0x558e8182c400 s 0x558e818c24e0 2019-12-07T19:33:52.751+0000 7fe2d0a78700 10 osd.0 15 heartbeat_reset closing (old) failed hb con 0x558e8182c400 2019-12-07T19:33:52.751+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e818ff800 0x558e818e7700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:52.751+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e818ff800 0x558e818e7700 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:52.751+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e81984000 0x558e81986000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:52.751+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e81984000 0x558e81986000 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:52.751+0000 7fe2e753b700 10 osd.0 15 ms_handle_authentication new session 0x558e818c31e0 con 0x558e818ff800 entity osd.3 addr 2019-12-07T19:33:52.751+0000 7fe2e753b700 10 osd.0 15 ms_handle_authentication session 0x558e818c31e0 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:52.751+0000 7fe2e6d3a700 10 osd.0 15 ms_handle_authentication new session 0x558e818e8340 con 0x558e81984000 entity osd.3 addr 2019-12-07T19:33:52.751+0000 7fe2e6d3a700 10 osd.0 15 ms_handle_authentication session 0x558e818e8340 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:52.751+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.131:0/37076 conn(0x558e818ff800 0x558e818e7700 crc :-1 s=READY pgs=13 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:52.751+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.131:0/37076 conn(0x558e81984000 0x558e81986000 crc :-1 s=READY pgs=13 cs=0 l=1 rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:52.779+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.829+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.841+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.5 172.21.15.109:0/36936 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.842324+0000/16.308647276s send_stamp 16.308647276s delta_ub 0.011922340s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e811f7800 con 0x558e8181b800 2019-12-07T19:33:52.841+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011364077s,0.011922340s]) 2019-12-07T19:33:52.841+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.842324+0000/16.308647276s send_stamp 16.297283199s delta_ub 18446744073.698187539s) v5 -- 0x558e811f6800 con 0x558e8181b800 2019-12-07T19:33:52.841+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.5 172.21.15.109:0/36936 5 ==== osd_ping(ping e15 up_from 13 ping_stamp 2019-12-07T19:33:52.842324+0000/16.308647276s send_stamp 16.308647276s delta_ub 0.011922340s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e811f7800 con 0x558e8182c800 2019-12-07T19:33:52.841+0000 7fe2e7d3c700 20 osd.0 15 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011318226s,0.011922340s]) 2019-12-07T19:33:52.841+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e15 up_from 13 ping_stamp 2019-12-07T19:33:52.842324+0000/16.308647276s send_stamp 16.297329050s delta_ub 18446744073.698233390s) v5 -- 0x558e818ee400 con 0x558e8182c800 2019-12-07T19:33:52.879+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.929+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:52.961+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e81984400 0x558e81986580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:52.961+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e81984400 0x558e81986580 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:52.961+0000 7fe2e6d3a700 10 osd.0 15 ms_handle_authentication new session 0x558e818e84e0 con 0x558e81984400 entity client.admin addr 2019-12-07T19:33:52.961+0000 7fe2e6d3a700 10 osd.0 15 ms_handle_authentication session 0x558e818e84e0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:52.961+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/481722811 conn(0x558e81984400 0x558e81986580 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=client.4295 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:52.961+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4295 172.21.15.131:0/481722811 1 ==== command(tid 1: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558e80231a40 con 0x558e81984400 2019-12-07T19:33:52.961+0000 7fe2d959f700 20 osd.0 15 OSD::ms_dispatch: command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:52.961+0000 7fe2d959f700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:52.961+0000 7fe2d959f700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:52.961+0000 7fe2d959f700 20 osd.0 15 _dispatch 0x558e80231a40 command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:52.962+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/481722811 -- command_reply(tid 1: 0 ) v1 -- 0x558e8022e380 con 0x558e81984400 2019-12-07T19:33:52.980+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.005+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4295 172.21.15.131:0/481722811 2 ==== command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558e80231e00 con 0x558e81984400 2019-12-07T19:33:53.005+0000 7fe2d959f700 20 osd.0 15 OSD::ms_dispatch: command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:33:53.005+0000 7fe2d959f700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:53.005+0000 7fe2d959f700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:53.005+0000 7fe2d959f700 20 osd.0 15 _dispatch 0x558e80231e00 command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:33:53.007+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/481722811 -- command_reply(tid 2: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558e8022f340 con 0x558e81984400 2019-12-07T19:33:53.030+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.080+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.130+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.180+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.230+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 41140 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.237+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 13 ==== osd_map(15..16 src has 1..16) v4 ==== 829+0+0 (crc 0 0 0) 0x558e817bbb00 con 0x558e80f39000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 osd.0 15 OSD::ms_dispatch: osd_map(15..16 src has 1..16) v4 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 osd.0 15 do_waiters -- start 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 osd.0 15 do_waiters -- finish 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 osd.0 15 _dispatch 0x558e817bbb00 osd_map(15..16 src has 1..16) v4 2019-12-07T19:33:53.237+0000 7fe2d959f700 3 osd.0 15 handle_osd_map epochs [15,16], i have 15, src has [1,16] 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 osd.0 15 handle_osd_map got inc map for epoch 16 2019-12-07T19:33:53.237+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3630e826:::osdmap.15:0# 0x0~0 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1168 size 0x1168 (4456) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866b60 blob([0x23000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e81789f80 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e80f04380 blob([!~1000,0x24000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,168]) SharedBlob(0x558e818676c0 sbid 0x0)) need 0x1000~168 cache has 0x[1000~168] 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3630e826:::osdmap.15:0# 0x0~1168 = 4456 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 osd.0 15 add_map_bl 15 4456 bytes 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 osd.0 15 got_full_map 16, nothing requested 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 osd.0 15 handle_osd_map pg_num_history pg_num_history(e16 pg_nums {} deleted_pools ) 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps purged_snaps {16={}} 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 osd.0 15 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e16 [1,16] lci=[13,16]) 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e817de600 seq 9 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c423972f:::inc_osdmap.16:0# key 0x7f7fffffffffffffffc42397'/!inc_osdmap.16!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:53.237+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c423972f:::inc_osdmap.16:0# 0x0~1b9 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1059 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c423972f:::inc_osdmap.16:0# 0x0~1b9 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~1b9 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xe47 on front/back, now 0x0~1000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e817de600 1 blobs 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x25000~1000/1000,1000,0 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x25000~1000] 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81867500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81866070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81867500 blob([0x25000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81866070 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.blob(0x558e81867500) get_ref 0x0~1b9 Blob(0x558e81867500 blob([0x25000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81866070 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.blob(0x558e81867500) get_ref init 0x1000, 1000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1b9: 0x0~1b9 Blob(0x558e81867500 blob([0x25000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1b9) SharedBlob(0x558e81866070 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81866088 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1b9 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9a08) dirty_range mark inline shard dirty 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c423972f:::inc_osdmap.16:0# 0x0~1b9 = 0 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3cb0e826:::osdmap.16:0# key 0x7f7fffffffffffffff3cb0e8'&!osdmap.16!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:53.237+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3cb0e826:::osdmap.16:0# 0x0~1235 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1060 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3cb0e826:::osdmap.16:0# 0x0~1235 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~235 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xdcb on front/back, now 0x0~1000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e817de600 2 blobs 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x26000~2000/1000,2000,0 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x26000~2000] 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e811d1730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d08c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e811d1730 blob([0x26000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d08c0 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.blob(0x558e811d1730) get_ref 0x0~1000 Blob(0x558e811d1730 blob([0x26000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d08c0 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.blob(0x558e811d1730) get_ref init 0x1000, 1000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e811d1730 blob([0x26000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e811d08c0 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e811d08d8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81079260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d0000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81079260 blob([!~1000,0x27000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d0000 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.blob(0x558e81079260) get_ref 0x1000~235 Blob(0x558e81079260 blob([!~1000,0x27000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e811d0000 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.blob(0x558e81079260) get_ref init 0x2000, 1000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~235: 0x1000~235 Blob(0x558e81079260 blob([!~1000,0x27000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,235]) SharedBlob(0x558e811d0000 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e811d0018 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1235 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9c88) dirty_range mark inline shard dirty 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3cb0e826:::osdmap.16:0# 0x0~1235 = 0 2019-12-07T19:33:53.237+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~12 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfee 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~12: 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e817de600 0 blobs 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:53.237+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:53.237+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:53.237+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e817de600 0 blobs 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e817de600 cost 22563 (4 ios * 4000 + 6563 bytes) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e817de600 onodes 0x558e80251b80,0x558e817b9180,0x558e817b9900,0x558e817b9b80 shared_blobs 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9a08) update #-1:c423972f:::inc_osdmap.16:0# 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9a08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c423972f:::inc_osdmap.16:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9c88) update #-1:3cb0e826:::osdmap.16:0# 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9c88) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3cb0e826:::osdmap.16:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e817de600 allocated 0x[25000~3000] released 0x[] 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 freelist allocate 0x25000~3000 2019-12-07T19:33:53.237+0000 7fe2d959f700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de600 prepare 2019-12-07T19:33:53.237+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e817de600 2019-12-07T19:33:53.237+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de600 aio_wait 2019-12-07T19:33:53.237+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e817de600 2019-12-07T19:33:53.237+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de600 io_done 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:53.237+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8788 < min_flush_size 524288 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9900 had 1 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9b80 had 1 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:33:53.237+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:53.237+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:53.237+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8835 < min_flush_size 524288 2019-12-07T19:33:53.237+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8835 < min_flush_size 524288 2019-12-07T19:33:53.237+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:53.237+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xa528~2283 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:53.237+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xa528 0xa528~2283 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xa528 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x528 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x7ab and padding block with 0x855 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc7ab 2019-12-07T19:33:53.237+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:53.237+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:53.237+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 1 in 0.000175504s (0.000033193s flush + 0.000142311s kv commit) 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:53.237+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e817de600> 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <0x558e811de000> 2019-12-07T19:33:53.237+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de600 kv_submitted 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e817de600 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e817de600 osr 0x558e8107a500 2019-12-07T19:33:53.237+0000 7fe2d7285700 10 osd.0 15 _committed_osd_maps 16..16 2019-12-07T19:33:53.237+0000 7fe2d7285700 10 osd.0 15 advance to epoch 16 (<= last 16 <= newest_map 16) 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e81795e00) prepare_write seq 7 0x1b000~1000 crc 24ecfc60 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e81795e00) prepare_write seq 7 0x2000~1000 crc d4738fd8 2019-12-07T19:33:53.237+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de300 deferred_cleanup 2019-12-07T19:33:53.237+0000 7fe2d7285700 7 osd.0 16 consume_map version 16 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e817de300 onodes 0x558e80251b80,0x558e817b9180,0x558e8191a000,0x558e8191a280 2019-12-07T19:33:53.237+0000 7fe2d7285700 20 osd.0 16 prune_pg_created 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81866fd8 in 0x558e80219340) _finish_write discard buffer(0x558e811718c0 space 0x558e81866fd8 0x0~1000 writing nocache) 2019-12-07T19:33:53.237+0000 7fe2d7285700 10 osd.0:0.consume_map 16 (was 15) 2019-12-07T19:33:53.237+0000 7fe2d7285700 10 osd.0:1.consume_map 16 (was 15) 2019-12-07T19:33:53.237+0000 7fe2d7285700 10 osd.0:2.consume_map 16 (was 15) 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e818670b8 in 0x558e80219340) _finish_write discard buffer(0x558e81171d40 space 0x558e818670b8 0x0~1000 writing nocache) 2019-12-07T19:33:53.237+0000 7fe2d7285700 10 osd.0:3.consume_map 16 (was 15) 2019-12-07T19:33:53.237+0000 7fe2d7285700 10 osd.0:4.consume_map 16 (was 15) 2019-12-07T19:33:53.237+0000 7fe2d7285700 10 osd.0:5.consume_map 16 (was 15) 2019-12-07T19:33:53.237+0000 7fe2d7285700 10 osd.0:6.consume_map 16 (was 15) 2019-12-07T19:33:53.237+0000 7fe2d7285700 10 osd.0:7.consume_map 16 (was 15) 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81867358 in 0x558e80219340) _finish_write discard buffer(0x558e81171dd0 space 0x558e81867358 0x1000~1000 writing nocache) 2019-12-07T19:33:53.237+0000 7fe2d7285700 10 osd.0 16 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e817de300 done 2019-12-07T19:33:53.237+0000 7fe2d7285700 7 osd.0 16 activate_map version 16 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c300 deferred_cleanup 2019-12-07T19:33:53.237+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e817de300 [] 2019-12-07T19:33:53.237+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:53.237+0000 7fe2cc26f700 20 osd.0 16 agent_entry empty queue 2019-12-07T19:33:53.237+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 deferred_cleanup 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c300 onodes 0x558e80251b80,0x558e817b9180,0x558e8191a500,0x558e8191a780 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81789f98 in 0x558e80219340) _finish_write discard buffer(0x558e81170f30 space 0x558e81789f98 0x0~1000 writing nocache) 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e818665c8 in 0x558e80219340) _finish_write discard buffer(0x558e81170ea0 space 0x558e818665c8 0x0~1000 writing nocache) 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e818676d8 in 0x558e80219340) _finish_write discard buffer(0x558e81170fc0 space 0x558e818676d8 0x1000~1000 writing nocache) 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c300 done 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e817de600 deferred_queued 2019-12-07T19:33:53.237+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c300 [] 2019-12-07T19:33:53.237+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:53.237+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:53.280+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.330+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.380+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.413+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/481722811 conn(0x558e81984400 msgr2=0x558e81986580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 90 2019-12-07T19:33:53.413+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/481722811 conn(0x558e81984400 msgr2=0x558e81986580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:53.413+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/481722811 conn(0x558e81984400 0x558e81986580 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:53.413+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/481722811 conn(0x558e81984400 0x558e81986580 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:53.413+0000 7fe2d959f700 2 osd.0 16 ms_handle_reset con 0x558e81984400 session 0x558e818e84e0 2019-12-07T19:33:53.430+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.480+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.530+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.531+0000 7fe2e2ca2700 10 osd.0 16 tick_without_osd_lock 2019-12-07T19:33:53.531+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffd8000/0x40000000/0x1680000000, data 0xac74/0x26000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:53.531+0000 7fe2e2ca2700 25 osd.0 16 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153258+0000 last_rx_front 2019-12-07T19:33:51.153336+0000 2019-12-07T19:33:53.531+0000 7fe2e2ca2700 25 osd.0 16 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153377+0000 last_rx_front 2019-12-07T19:33:51.153481+0000 2019-12-07T19:33:53.531+0000 7fe2e2ca2700 25 osd.0 16 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153242+0000 last_rx_front 2019-12-07T19:33:51.153313+0000 2019-12-07T19:33:53.531+0000 7fe2e2ca2700 25 osd.0 16 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153637+0000 last_rx_front 2019-12-07T19:33:51.153527+0000 2019-12-07T19:33:53.531+0000 7fe2e2ca2700 25 osd.0 16 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153392+0000 last_rx_front 2019-12-07T19:33:51.153509+0000 2019-12-07T19:33:53.531+0000 7fe2e2ca2700 25 osd.0 16 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153611+0000 last_rx_front 2019-12-07T19:33:51.153694+0000 2019-12-07T19:33:53.531+0000 7fe2e2ca2700 25 osd.0 16 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153670+0000 last_rx_front 2019-12-07T19:33:51.153496+0000 2019-12-07T19:33:53.531+0000 7fe2e2ca2700 20 osd.0 16 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:53.532+0000 7fe2e2ca2700 20 osd.0 16 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:53.532+0000 7fe2e2ca2700 20 osd.0 16 scrub_load_below_threshold loadavg per cpu 0.07 < max 5 = yes 2019-12-07T19:33:53.532+0000 7fe2e2ca2700 20 osd.0 16 sched_scrub load_is_low=1 2019-12-07T19:33:53.532+0000 7fe2e2ca2700 20 osd.0 16 sched_scrub done 2019-12-07T19:33:53.532+0000 7fe2e2ca2700 10 osd.0 16 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:53.532+0000 7fe2e2ca2700 20 osd.0 16 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:53.532+0000 7fe2e2ca2700 10 osd.0 16 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:53.581+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.631+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.652+0000 7fe2e4535700 10 osd.0 16 tick 2019-12-07T19:33:53.652+0000 7fe2e4535700 10 osd.0 16 do_waiters -- start 2019-12-07T19:33:53.652+0000 7fe2e4535700 10 osd.0 16 do_waiters -- finish 2019-12-07T19:33:53.652+0000 7fe2e4535700 20 osd.0 16 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:53.681+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.731+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.781+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.831+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.881+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.931+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:53.981+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.031+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.082+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.132+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.182+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.232+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.282+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.332+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.382+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.432+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.482+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.495+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:33:54.532+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.570+0000 7fe2e2ca2700 10 osd.0 16 tick_without_osd_lock 2019-12-07T19:33:54.570+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffd8000/0x40000000/0x1680000000, data 0xac74/0x26000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:54.570+0000 7fe2e2ca2700 25 osd.0 16 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153258+0000 last_rx_front 2019-12-07T19:33:51.153336+0000 2019-12-07T19:33:54.570+0000 7fe2e2ca2700 25 osd.0 16 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153377+0000 last_rx_front 2019-12-07T19:33:51.153481+0000 2019-12-07T19:33:54.570+0000 7fe2e2ca2700 25 osd.0 16 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153242+0000 last_rx_front 2019-12-07T19:33:51.153313+0000 2019-12-07T19:33:54.570+0000 7fe2e2ca2700 25 osd.0 16 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153637+0000 last_rx_front 2019-12-07T19:33:51.153527+0000 2019-12-07T19:33:54.570+0000 7fe2e2ca2700 25 osd.0 16 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153392+0000 last_rx_front 2019-12-07T19:33:51.153509+0000 2019-12-07T19:33:54.570+0000 7fe2e2ca2700 25 osd.0 16 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153611+0000 last_rx_front 2019-12-07T19:33:51.153694+0000 2019-12-07T19:33:54.570+0000 7fe2e2ca2700 25 osd.0 16 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153670+0000 last_rx_front 2019-12-07T19:33:51.153496+0000 2019-12-07T19:33:54.570+0000 7fe2e2ca2700 20 osd.0 16 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:54.570+0000 7fe2e2ca2700 10 osd.0 16 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:54.570+0000 7fe2e2ca2700 20 osd.0 16 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:54.570+0000 7fe2e2ca2700 10 osd.0 16 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:54.582+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 1 osrs, 1 txcs 2019-12-07T19:33:54.582+0000 7fe2dc28f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock osr 0x558e8107a500 2 ios pending 2019-12-07T19:33:54.582+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 7 0x2000~1000 2019-12-07T19:33:54.582+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x2000~1000 crc d4738fd8 2019-12-07T19:33:54.582+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 7 0x1b000~1000 2019-12-07T19:33:54.582+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x1b000~1000 crc 24ecfc60 2019-12-07T19:33:54.583+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.583+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish osr 0x558e8107a500 2019-12-07T19:33:54.583+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish dequeueing 2019-12-07T19:33:54.633+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 43208 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.636+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.2 172.21.15.131:0/37077 5 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.637021+0000/18.108739871s send_stamp 18.108739871s delta_ub 0.017155099s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e811f7600 con 0x558e8182d400 2019-12-07T19:33:54.636+0000 7fe2e7d3c700 20 osd.0 16 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016896087s,0.017155099s]) 2019-12-07T19:33:54.636+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e16 up_from 13 ping_stamp 2019-12-07T19:33:54.637021+0000/18.108739871s send_stamp 18.091843784s delta_ub 18446744073.692655529s) v5 -- 0x558e818ee400 con 0x558e8182d400 2019-12-07T19:33:54.636+0000 7fe2e7d3c700 10 osd.0 16 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:54.636+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.2 172.21.15.131:0/37077 5 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.637021+0000/18.108739871s send_stamp 18.108739871s delta_ub 0.017155099s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e811f7400 con 0x558e8182c000 2019-12-07T19:33:54.636+0000 7fe2e6d3a700 20 osd.0 16 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016835418s,0.017155099s]) 2019-12-07T19:33:54.636+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e16 up_from 13 ping_stamp 2019-12-07T19:33:54.637021+0000/18.108739871s send_stamp 18.091904453s delta_ub 18446744073.692716198s) v5 -- 0x558e818ef200 con 0x558e8182c000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 2 ==== osd_map(17..17 src has 1..17) v4 ==== 1311+0+0 (unknown 0 0 0) 0x558e80238fc0 con 0x558e818d6000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 osd.0 16 OSD::ms_dispatch: osd_map(17..17 src has 1..17) v4 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 osd.0 16 do_waiters -- start 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 osd.0 16 do_waiters -- finish 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 osd.0 16 _dispatch 0x558e80238fc0 osd_map(17..17 src has 1..17) v4 2019-12-07T19:33:54.637+0000 7fe2d4a80700 3 osd.0 16 handle_osd_map epochs [17,17], i have 16, src has [1,17] 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 osd.0 16 handle_osd_map got inc map for epoch 17 2019-12-07T19:33:54.637+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3cb0e826:::osdmap.16:0# 0x0~0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1235 size 0x1235 (4661) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d1730 blob([0x26000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e811d08c0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81079260 blob([!~1000,0x27000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,235]) SharedBlob(0x558e811d0000 sbid 0x0)) need 0x1000~235 cache has 0x[1000~235] 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3cb0e826:::osdmap.16:0# 0x0~1235 = 4661 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 osd.0 16 add_map_bl 16 4661 bytes 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 osd.0 16 got_full_map 17, nothing requested 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 osd.0 16 handle_osd_map pg_num_history pg_num_history(e17 pg_nums {} deleted_pools ) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps purged_snaps {17={}} 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 osd.0 16 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e17 [1,17] lci=[13,17]) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e8106c900 seq 10 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:cfa3972f:::inc_osdmap.17:0# key 0x7f7fffffffffffffffcfa397'/!inc_osdmap.17!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:cfa3972f:::inc_osdmap.17:0# 0x0~4f3 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1061 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:cfa3972f:::inc_osdmap.17:0# 0x0~4f3 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~4f3 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xb0d on front/back, now 0x0~1000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c900 1 blobs 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x28000~1000/1000,1000,0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x28000~1000] 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e8000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e8070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e8000 blob([0x28000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8070 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.blob(0x558e817e8000) get_ref 0x0~4f3 Blob(0x558e817e8000 blob([0x28000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8070 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.blob(0x558e817e8000) get_ref init 0x1000, 1000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~4f3: 0x0~4f3 Blob(0x558e817e8000 blob([0x28000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4f3) SharedBlob(0x558e817e8070 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e817e8088 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x4f3 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191ab08) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:cfa3972f:::inc_osdmap.17:0# 0x0~4f3 = 0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3340e826:::osdmap.17:0# key 0x7f7fffffffffffffff3340e8'&!osdmap.17!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3340e826:::osdmap.17:0# 0x0~1292 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1062 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3340e826:::osdmap.17:0# 0x0~1292 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~292 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xd6e on front/back, now 0x0~1000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c900 2 blobs 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x29000~2000/1000,2000,0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x29000~2000] 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e8150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e81c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e8150 blob([0x29000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e81c0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.blob(0x558e817e8150) get_ref 0x0~1000 Blob(0x558e817e8150 blob([0x29000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e81c0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.blob(0x558e817e8150) get_ref init 0x1000, 1000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e817e8150 blob([0x29000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e817e81c0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e817e81d8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e8230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e82a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e8230 blob([!~1000,0x2a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e82a0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.blob(0x558e817e8230) get_ref 0x1000~292 Blob(0x558e817e8230 blob([!~1000,0x2a000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e817e82a0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.blob(0x558e817e8230) get_ref init 0x2000, 1000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~292: 0x1000~292 Blob(0x558e817e8230 blob([!~1000,0x2a000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,292]) SharedBlob(0x558e817e82a0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e817e82b8 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1292 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191ad88) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3340e826:::osdmap.17:0# 0x0~1292 = 0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~12 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~12 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfee 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~12: 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c900 0 blobs 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~12 = 0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c900 0 blobs 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c900 cost 23482 (4 ios * 4000 + 7482 bytes) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c900 onodes 0x558e80251b80,0x558e817b9180,0x558e8191aa00,0x558e8191ac80 shared_blobs 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update inline shard 18 bytes from 1 extents 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 45 (21 bytes onode + 2 bytes spanning blobs + 22 bytes inline extents) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191ab08) update #-1:cfa3972f:::inc_osdmap.17:0# 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191ab08) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:cfa3972f:::inc_osdmap.17:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191ad88) update #-1:3340e826:::osdmap.17:0# 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191ad88) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3340e826:::osdmap.17:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c900 allocated 0x[28000~3000] released 0x[] 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 freelist allocate 0x28000~3000 2019-12-07T19:33:54.637+0000 7fe2d4a80700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 prepare 2019-12-07T19:33:54.637+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8106c900 2019-12-07T19:33:54.637+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 aio_wait 2019-12-07T19:33:54.637+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c900 2019-12-07T19:33:54.637+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 io_done 2019-12-07T19:33:54.637+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:54.637+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:33:54.637+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:54.637+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8788 < min_flush_size 524288 2019-12-07T19:33:54.637+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:33:54.637+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:33:54.637+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8191aa00 had 1 2019-12-07T19:33:54.637+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8191ac80 had 1 2019-12-07T19:33:54.637+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:33:54.637+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3860791296 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:54.638+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998779) bluestore 86 GiB free (0.959998), bluefs_ratio 0.0399536 2019-12-07T19:33:54.638+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:33:54.638+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:33:54.638+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:54.638+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc7ab~2277 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:54.638+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc7ab 0xc7ab~2277 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:54.638+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:54.638+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xc7ab 2019-12-07T19:33:54.638+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x7ab 2019-12-07T19:33:54.638+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:54.638+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xa22 and padding block with 0x5de 2019-12-07T19:33:54.638+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xea22 2019-12-07T19:33:54.638+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:54.638+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:54.638+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:54.638+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:54.638+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 1 in 0.000217000s (0.000055164s flush + 0.000161836s kv commit) 2019-12-07T19:33:54.638+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:54.638+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:54.638+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8106c900> 2019-12-07T19:33:54.638+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <0x558e81795e00> 2019-12-07T19:33:54.638+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 kv_submitted 2019-12-07T19:33:54.638+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c900 2019-12-07T19:33:54.638+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e8106c900 osr 0x558e8107a500 2019-12-07T19:33:54.638+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de3c0) prepare_write seq 8 0x1b000~1000 crc 3e33509d 2019-12-07T19:33:54.638+0000 7fe2d7285700 10 osd.0 16 _committed_osd_maps 17..17 2019-12-07T19:33:54.638+0000 7fe2d7285700 10 osd.0 16 advance to epoch 17 (<= last 17 <= newest_map 17) 2019-12-07T19:33:54.638+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de3c0) prepare_write seq 8 0x2000~1000 crc 6f489be5 2019-12-07T19:33:54.638+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e817de600 deferred_cleanup 2019-12-07T19:33:54.638+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e817de600 onodes 0x558e80251b80,0x558e817b9180,0x558e817b9900,0x558e817b9b80 2019-12-07T19:33:54.638+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d0018 in 0x558e80219340) _finish_write discard buffer(0x558e81171710 space 0x558e811d0018 0x1000~1000 writing nocache) 2019-12-07T19:33:54.638+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d08d8 in 0x558e80219340) _finish_write discard buffer(0x558e81171680 space 0x558e811d08d8 0x0~1000 writing nocache) 2019-12-07T19:33:54.638+0000 7fe2d7285700 7 osd.0 17 consume_map version 17 2019-12-07T19:33:54.638+0000 7fe2d7285700 20 osd.0 17 prune_pg_created 2019-12-07T19:33:54.638+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81866088 in 0x558e80219340) _finish_write discard buffer(0x558e811715f0 space 0x558e81866088 0x0~1000 writing nocache) 2019-12-07T19:33:54.638+0000 7fe2d7285700 10 osd.0:0.consume_map 17 (was 16) 2019-12-07T19:33:54.638+0000 7fe2d7285700 10 osd.0:1.consume_map 17 (was 16) 2019-12-07T19:33:54.638+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e817de600 done 2019-12-07T19:33:54.638+0000 7fe2d7285700 10 osd.0:2.consume_map 17 (was 16) 2019-12-07T19:33:54.638+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c900 deferred_queued 2019-12-07T19:33:54.638+0000 7fe2d7285700 10 osd.0:3.consume_map 17 (was 16) 2019-12-07T19:33:54.638+0000 7fe2d7285700 10 osd.0:4.consume_map 17 (was 16) 2019-12-07T19:33:54.638+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e817de600 [] 2019-12-07T19:33:54.638+0000 7fe2d7285700 10 osd.0:5.consume_map 17 (was 16) 2019-12-07T19:33:54.638+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:54.638+0000 7fe2d7285700 10 osd.0:6.consume_map 17 (was 16) 2019-12-07T19:33:54.638+0000 7fe2d7285700 10 osd.0:7.consume_map 17 (was 16) 2019-12-07T19:33:54.638+0000 7fe2d7285700 10 osd.0 17 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:54.638+0000 7fe2d7285700 7 osd.0 17 activate_map version 17 2019-12-07T19:33:54.638+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:54.638+0000 7fe2cc26f700 20 osd.0 17 agent_entry empty queue 2019-12-07T19:33:54.683+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.698+0000 7fe2e4535700 10 osd.0 17 tick 2019-12-07T19:33:54.698+0000 7fe2e4535700 10 osd.0 17 do_waiters -- start 2019-12-07T19:33:54.698+0000 7fe2e4535700 10 osd.0 17 do_waiters -- finish 2019-12-07T19:33:54.698+0000 7fe2e4535700 20 osd.0 17 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:54.733+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.750+0000 7fe2e7d3c700 0 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.131:0/37078 conn(0x558e81749400 msgr2=0x558e81817b80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2019-12-07T19:33:54.750+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.1 172.21.15.131:0/37078 4 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.751090+0000/18.208013836s send_stamp 18.208013836s delta_ub 0.002411017s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e811f6000 con 0x558e8182dc00 2019-12-07T19:33:54.750+0000 7fe2e6d3a700 20 osd.0 17 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002110541s,0.002411017s]) 2019-12-07T19:33:54.750+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.751090+0000/18.208013836s send_stamp 18.205903295s delta_ub 18446744073.707441075s) v5 -- 0x558e818ef200 con 0x558e8182dc00 2019-12-07T19:33:54.750+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.1 172.21.15.131:0/37078 4 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.751090+0000/18.208013836s send_stamp 18.208013836s delta_ub 0.002411017s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e811f7800 con 0x558e81749400 2019-12-07T19:33:54.750+0000 7fe2e6d3a700 10 osd.0 17 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:54.750+0000 7fe2e7d3c700 20 osd.0 17 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.001990679s,0.002411017s]) 2019-12-07T19:33:54.750+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.751090+0000/18.208013836s send_stamp 18.206023157s delta_ub 18446744073.707560937s) v5 -- 0x558e818ee400 con 0x558e81749400 2019-12-07T19:33:54.750+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.131:0/37078 conn(0x558e81749400 msgr2=0x558e81817b80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 64 2019-12-07T19:33:54.750+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.131:0/37078 conn(0x558e81749400 msgr2=0x558e81817b80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:54.750+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.131:0/37078 conn(0x558e81749400 0x558e81817b80 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:54.750+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.131:0/37078 conn(0x558e81749400 0x558e81817b80 crc :-1 s=READY pgs=1 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:54.750+0000 7fe2d0a78700 20 osd.0 17 heartbeat_reset con 0x558e81749400 s 0x558e818c2820 2019-12-07T19:33:54.750+0000 7fe2d0a78700 10 osd.0 17 heartbeat_reset closing (old) failed hb con 0x558e81749400 2019-12-07T19:33:54.750+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e81984c00 0x558e81986b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:54.750+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e81985000 0x558e81987080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:54.750+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.131:0/37078 conn(0x558e8182dc00 msgr2=0x558e81845600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 83 2019-12-07T19:33:54.750+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.131:0/37078 conn(0x558e8182dc00 msgr2=0x558e81845600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:54.751+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e81985000 0x558e81987080 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:54.751+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.131:0/37078 conn(0x558e8182dc00 0x558e81845600 crc :-1 s=READY pgs=4 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:54.751+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.131:0/37078 conn(0x558e8182dc00 0x558e81845600 crc :-1 s=READY pgs=4 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:54.751+0000 7fe2d1a7a700 20 osd.0 17 heartbeat_reset con 0x558e8182dc00 s 0x558e81807ba0 2019-12-07T19:33:54.751+0000 7fe2d1a7a700 10 osd.0 17 heartbeat_reset closing (old) failed hb con 0x558e8182dc00 2019-12-07T19:33:54.751+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e81984c00 0x558e81986b00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:54.751+0000 7fe2e6d3a700 10 osd.0 17 ms_handle_authentication new session 0x558e818e8680 con 0x558e81984c00 entity osd.1 addr 2019-12-07T19:33:54.751+0000 7fe2e7d3c700 10 osd.0 17 ms_handle_authentication new session 0x558e818e8820 con 0x558e81985000 entity osd.1 addr 2019-12-07T19:33:54.751+0000 7fe2e7d3c700 10 osd.0 17 ms_handle_authentication session 0x558e818e8820 osd.1 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:54.751+0000 7fe2e6d3a700 10 osd.0 17 ms_handle_authentication session 0x558e818e8680 osd.1 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:54.751+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.131:0/37078 conn(0x558e81984c00 0x558e81986b00 crc :-1 s=READY pgs=8 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:54.751+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.131:0/37078 conn(0x558e81985000 0x558e81987080 crc :-1 s=READY pgs=8 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:54.771+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.7 172.21.15.109:0/36939 4 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.690797630s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8200 con 0x558e81749000 2019-12-07T19:33:54.771+0000 7fe2e6d3a700 20 osd.0 17 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690042410s,18446744073.690797630s]) 2019-12-07T19:33:54.771+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.227158369s delta_ub 0.019509206s) v5 -- 0x558e818ef200 con 0x558e81749000 2019-12-07T19:33:54.771+0000 7fe2e6d3a700 10 osd.0 17 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:54.771+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.7 172.21.15.109:0/36939 4 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.207649163s delta_ub 18446744073.690797630s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8200 con 0x558e81748c00 2019-12-07T19:33:54.771+0000 7fe2e6d3a700 20 osd.0 17 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690008470s,18446744073.690797630s]) 2019-12-07T19:33:54.771+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:54.772210+0000/18.207649163s send_stamp 18.227192309s delta_ub 0.019543146s) v5 -- 0x558e811f6a00 con 0x558e81748c00 2019-12-07T19:33:54.783+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.833+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.883+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.894+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:33:54.933+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:54.983+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.033+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.083+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.134+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.159+0000 7fe2e6d3a700 0 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.109:0/36938 conn(0x558e8182d000 msgr2=0x558e81844580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2019-12-07T19:33:55.159+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.4 172.21.15.109:0/36938 6 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:55.159452+0000/18.608678189s send_stamp 18.608678189s delta_ub 18446744073.704375639s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e8181bc00 2019-12-07T19:33:55.159+0000 7fe2e753b700 20 osd.0 17 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703756582s,18446744073.704375639s]) 2019-12-07T19:33:55.159+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:55.159452+0000/18.608678189s send_stamp 18.614473223s delta_ub 0.005795034s) v5 -- 0x558e811f7a00 con 0x558e8181bc00 2019-12-07T19:33:55.159+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.4 172.21.15.109:0/36938 6 ==== osd_ping(ping e17 up_from 13 ping_stamp 2019-12-07T19:33:55.159452+0000/18.608678189s send_stamp 18.608678189s delta_ub 18446744073.704375639s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e811f7400 con 0x558e8182d000 2019-12-07T19:33:55.159+0000 7fe2e753b700 10 osd.0 17 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 15 -> 17 (as per caller) 2019-12-07T19:33:55.159+0000 7fe2e6d3a700 20 osd.0 17 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703695911s,18446744073.704375639s]) 2019-12-07T19:33:55.159+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e17 up_from 13 ping_stamp 2019-12-07T19:33:55.159452+0000/18.608678189s send_stamp 18.614533894s delta_ub 0.005855705s) v5 -- 0x558e811f6a00 con 0x558e8182d000 2019-12-07T19:33:55.159+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.109:0/36938 conn(0x558e8182d000 msgr2=0x558e81844580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 79 2019-12-07T19:33:55.159+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.109:0/36938 conn(0x558e8182d000 msgr2=0x558e81844580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:55.159+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.109:0/36938 conn(0x558e8182d000 0x558e81844580 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:55.159+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.109:0/36938 conn(0x558e8182d000 0x558e81844580 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:55.159+0000 7fe2d1a7a700 20 osd.0 17 heartbeat_reset con 0x558e8182d000 s 0x558e818076c0 2019-12-07T19:33:55.159+0000 7fe2d1a7a700 10 osd.0 17 heartbeat_reset closing (old) failed hb con 0x558e8182d000 2019-12-07T19:33:55.159+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.109:0/36938 conn(0x558e8181bc00 msgr2=0x558e81818680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 66 2019-12-07T19:33:55.159+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.109:0/36938 conn(0x558e8181bc00 msgr2=0x558e81818680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:55.159+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.109:0/36938 conn(0x558e8181bc00 0x558e81818680 crc :-1 s=READY pgs=4 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:55.159+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.109:0/36938 conn(0x558e8181bc00 0x558e81818680 crc :-1 s=READY pgs=4 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:55.159+0000 7fe2d0a78700 20 osd.0 17 heartbeat_reset con 0x558e8181bc00 s 0x558e818c2000 2019-12-07T19:33:55.159+0000 7fe2d0a78700 10 osd.0 17 heartbeat_reset closing (old) failed hb con 0x558e8181bc00 2019-12-07T19:33:55.159+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e81985400 0x558e81987600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:55.159+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e81985800 0x558e81987b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:55.159+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> conn(0x558e81985400 0x558e81987600 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:55.159+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> conn(0x558e81985800 0x558e81987b80 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:55.159+0000 7fe2e6d3a700 10 osd.0 17 ms_handle_authentication new session 0x558e818c2340 con 0x558e81985400 entity osd.4 addr 2019-12-07T19:33:55.159+0000 7fe2e7d3c700 10 osd.0 17 ms_handle_authentication new session 0x558e818c3a00 con 0x558e81985800 entity osd.4 addr 2019-12-07T19:33:55.159+0000 7fe2e7d3c700 10 osd.0 17 ms_handle_authentication session 0x558e818c3a00 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:55.159+0000 7fe2e6d3a700 10 osd.0 17 ms_handle_authentication session 0x558e818c2340 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:55.159+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.109:0/36938 conn(0x558e81985800 0x558e81987b80 crc :-1 s=READY pgs=13 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:55.159+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.109:0/36938 conn(0x558e81985400 0x558e81987600 crc :-1 s=READY pgs=9 cs=0 l=1 rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:55.184+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.196+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:33:55.234+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.284+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.334+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.384+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.434+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.484+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.521+0000 7fe2d5281700 20 osd.0 17 reports for 0 queries 2019-12-07T19:33:55.521+0000 7fe2d5281700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.0 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558e81175180 con 0x558e81748000 2019-12-07T19:33:55.521+0000 7fe2d5281700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(0 pgs tid 0 v 0) v2 -- 0x558e81175500 con 0x558e81748000 2019-12-07T19:33:55.534+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.584+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 45860 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.610+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 14 ==== pg_create2(e18 {1.2s0=18,2019-12-07T19:33:55.603918+0000}) v2 ==== 207+0+0 (crc 0 0 0) 0x558e819098c0 con 0x558e80f39000 2019-12-07T19:33:55.610+0000 7fe2e7d3c700 7 osd.0 17 handle_fast_pg_create pg_create2(e18 {1.2s0=18,2019-12-07T19:33:55.603918+0000}) v2 from mon.2 2019-12-07T19:33:55.610+0000 7fe2e7d3c700 20 osd.0 17 handle_fast_pg_create 1.2s0 e18@2019-12-07T19:33:55.603918+0000 history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 pi ([0,0] all_participants= intervals=) 2019-12-07T19:33:55.610+0000 7fe2e7d3c700 15 osd.0 17 enqueue_peering_evt 1.2s0 epoch_sent: 18 epoch_requested: 18 NullEvt +create_info 2019-12-07T19:33:55.610+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.610+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.610+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.610+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.610+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.610+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 map 18 > 17 2019-12-07T19:33:55.610+0000 7fe2caa6c700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 18, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.610+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:55.614+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 4 ==== pg_query2(1.ds5 1.ds5 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8191b680 con 0x558e818d6000 2019-12-07T19:33:55.614+0000 7fe2e753b700 15 osd.0 17 enqueue_peering_evt 1.ds5 epoch_sent: 18 epoch_requested: 18 MQuery 1.ds5 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.614+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.ds5 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.614+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.614+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.ds5 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.614+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.614+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.ds5 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.614+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 map 18 > 17 2019-12-07T19:33:55.614+0000 7fe2c9269700 20 osd.0 op_wq _add_slot_waiter 1.ds5 peering, item epoch is 18, will wait on OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.ds5 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.614+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:55.614+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 3 ==== osd_map(18..18 src has 1..18) v4 ==== 645+0+0 (unknown 0 0 0) 0x558e81908240 con 0x558e818d6000 2019-12-07T19:33:55.614+0000 7fe2d4a80700 20 osd.0 17 OSD::ms_dispatch: osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.614+0000 7fe2d4a80700 10 osd.0 17 do_waiters -- start 2019-12-07T19:33:55.614+0000 7fe2d4a80700 10 osd.0 17 do_waiters -- finish 2019-12-07T19:33:55.614+0000 7fe2d4a80700 20 osd.0 17 _dispatch 0x558e81908240 osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.615+0000 7fe2d4a80700 3 osd.0 17 handle_osd_map epochs [18,18], i have 17, src has [1,18] 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 osd.0 17 handle_osd_map got inc map for epoch 18 2019-12-07T19:33:55.615+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3340e826:::osdmap.17:0# 0x0~0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1292 size 0x1292 (4754) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8150 blob([0x29000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e817e81c0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8230 blob([!~1000,0x2a000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,292]) SharedBlob(0x558e817e82a0 sbid 0x0)) need 0x1000~292 cache has 0x[1000~292] 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3340e826:::osdmap.17:0# 0x0~1292 = 4754 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 osd.0 17 add_map_bl 17 4754 bytes 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 osd.0 17 got_full_map 18, nothing requested 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 osd.0 17 handle_osd_map recording new pool 1 pg_num 16 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 osd.0 17 handle_osd_map pg_num_history pg_num_history(e18 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps purged_snaps {18={}} 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 osd.0 17 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e18 [1,18] lci=[13,18]) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e819a0000 seq 11 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c15b972f:::inc_osdmap.18:0# key 0x7f7fffffffffffffffc15b97'/!inc_osdmap.18!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c15b972f:::inc_osdmap.18:0# 0x0~259 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1063 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c15b972f:::inc_osdmap.18:0# 0x0~259 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~259 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xda7 on front/back, now 0x0~1000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0000 1 blobs 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x2b000~1000/1000,1000,0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x2b000~1000] 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81788230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e83f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81788230 blob([0x2b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e83f0 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e81788230) get_ref 0x0~259 Blob(0x558e81788230 blob([0x2b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e83f0 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e81788230) get_ref init 0x1000, 1000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~259: 0x0~259 Blob(0x558e81788230 blob([0x2b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x259) SharedBlob(0x558e817e83f0 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e817e8408 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x259 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191b008) dirty_range mark inline shard dirty 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c15b972f:::inc_osdmap.18:0# 0x0~259 = 0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:39c0e826:::osdmap.18:0# key 0x7f7fffffffffffffff39c0e8'&!osdmap.18!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:39c0e826:::osdmap.18:0# 0x0~13ff 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1064 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:39c0e826:::osdmap.18:0# 0x0~13ff - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~3ff 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xc01 on front/back, now 0x0~1000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0000 2 blobs 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x2c000~2000/1000,2000,0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x2c000~2000] 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e84d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e8540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e84d0 blob([0x2c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8540 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e817e84d0) get_ref 0x0~1000 Blob(0x558e817e84d0 blob([0x2c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8540 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e817e84d0) get_ref init 0x1000, 1000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e817e84d0 blob([0x2c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e817e8540 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e817e8558 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e85b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e8620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e85b0 blob([!~1000,0x2d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8620 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e817e85b0) get_ref 0x1000~3ff Blob(0x558e817e85b0 blob([!~1000,0x2d000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e817e8620 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e817e85b0) get_ref init 0x2000, 1000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~3ff: 0x1000~3ff Blob(0x558e817e85b0 blob([!~1000,0x2d000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,3ff]) SharedBlob(0x558e817e8620 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e817e8638 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x13ff 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191b288) dirty_range mark inline shard dirty 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:39c0e826:::osdmap.18:0# 0x0~13ff = 0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x12 (18) bytes fadvise_flags 0x0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x12) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x38) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0000 0 blobs 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~12: 0x0~12 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x26 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0000 0 blobs 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0000 cost 23201 (4 ios * 4000 + 7201 bytes) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0000 onodes 0x558e80251b80,0x558e817b9180,0x558e8191af00,0x558e8191b180 shared_blobs 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191b008) update #-1:c15b972f:::inc_osdmap.18:0# 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191b008) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c15b972f:::inc_osdmap.18:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191b288) update #-1:39c0e826:::osdmap.18:0# 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191b288) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:39c0e826:::osdmap.18:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0000 allocated 0x[2b000~3000] released 0x[] 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 freelist allocate 0x2b000~3000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 prepare 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e819a0000 2019-12-07T19:33:55.615+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 1 ==== pg_query2(1.1s1 1.1s1 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8191b900 con 0x558e818d7000 2019-12-07T19:33:55.615+0000 7fe2e6d3a700 15 osd.0 17 enqueue_peering_evt 1.1s1 epoch_sent: 18 epoch_requested: 18 MQuery 1.1s1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.615+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.1s1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.615+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 2 ==== pg_query2(1.fs1 1.fs1 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8191b900 con 0x558e818d7000 2019-12-07T19:33:55.615+0000 7fe2e6d3a700 15 osd.0 17 enqueue_peering_evt 1.fs1 epoch_sent: 18 epoch_requested: 18 MQuery 1.fs1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.615+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.fs1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.1s1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.1s1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 map 18 > 17 2019-12-07T19:33:55.615+0000 7fe2cb26d700 20 osd.0 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 18, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.1s1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:55.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.fs1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.fs1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 map 18 > 17 2019-12-07T19:33:55.615+0000 7fe2c8267700 20 osd.0 op_wq _add_slot_waiter 1.fs1 peering, item epoch is 18, will wait on OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.fs1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:55.615+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 aio_wait 2019-12-07T19:33:55.615+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0000 2019-12-07T19:33:55.615+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 io_done 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:55.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5598 < min_flush_size 524288 2019-12-07T19:33:55.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8800 < min_flush_size 524288 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8191af00 had 1 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8191b180 had 1 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:33:55.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:33:55.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:33:55.615+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xea22~2277 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.615+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xea22 0xea22~2277 to file(ino 16 size 0x10000 mtime 2019-12-07T19:33:36.545530+0000 allocated 10000 extents [1:0xacd120000~10000]) 2019-12-07T19:33:55.615+0000 7fe2d7a86700 10 bluefs _allocate len 0xc99 from 0 2019-12-07T19:33:55.615+0000 7fe2d7a86700 10 bluefs _allocate len 0xc99 from 1 2019-12-07T19:33:55.615+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd130000 2019-12-07T19:33:55.615+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd0f0000~10000/10000,10000,acd130000 2019-12-07T19:33:55.615+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x20000 to include allocated 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 23 (was clean) 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd120000~10000 x_off 0xea22 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xa22 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc99 and padding block with 0x367 2019-12-07T19:33:55.615+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x20000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 3 ==== osd_map(16..18 src has 1..18) v4 ==== 2369+0+0 (unknown 0 0 0) 0x558e81908480 con 0x558e818d7000 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 osd.0 17 OSD::ms_dispatch: osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 osd.0 17 do_waiters -- start 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 osd.0 17 do_waiters -- finish 2019-12-07T19:33:55.615+0000 7fe2d4a80700 20 osd.0 17 _dispatch 0x558e81908480 osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.615+0000 7fe2d4a80700 3 osd.0 17 handle_osd_map epochs [16,18], i have 18, src has [1,18] 2019-12-07T19:33:55.615+0000 7fe2d4a80700 10 osd.0 17 no new maps here, dropping 2019-12-07T19:33:55.616+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x10c99 2019-12-07T19:33:55.616+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:55.616+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:55.616+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:55.616+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:55.616+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (23) on file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]), flushing log 2019-12-07T19:33:55.616+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:55.616+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:55.616+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 23 len 0x30 crc 0x3da8cabb) 2019-12-07T19:33:55.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x16000~1000 to file(ino 1 size 0x16000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:55.616+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x16000 0x16000~1000 to file(ino 1 size 0x16000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:55.616+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x17000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:55.616+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x16000 2019-12-07T19:33:55.616+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x17000 2019-12-07T19:33:55.616+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:55.616+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:33:55.616+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000019 2019-12-07T19:33:55.616+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:55.616+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 23 2019-12-07T19:33:55.616+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:55.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000521221s (0.000042185s flush + 0.000479036s kv commit) 2019-12-07T19:33:55.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:55.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:55.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a0000> 2019-12-07T19:33:55.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:55.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 kv_submitted 2019-12-07T19:33:55.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0000 2019-12-07T19:33:55.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e819a0000 osr 0x558e8107a500 2019-12-07T19:33:55.616+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de3c0) _discard drop 8 0x1b000~1000 2019-12-07T19:33:55.616+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de3c0) prepare_write seq 9 0x1b000~1000 crc 45ebe265 2019-12-07T19:33:55.616+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de3c0) _discard drop 8 0x2000~1000 2019-12-07T19:33:55.616+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de3c0) prepare_write seq 9 0x2000~1000 crc a7e9d153 2019-12-07T19:33:55.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:55.616+0000 7fe2d7285700 10 osd.0 17 _committed_osd_maps 18..18 2019-12-07T19:33:55.616+0000 7fe2d7285700 10 osd.0 17 advance to epoch 18 (<= last 18 <= newest_map 18) 2019-12-07T19:33:55.616+0000 7fe2d7285700 0 osd.0 18 crush map has features 288514119978713088, adjusting msgr requires for clients 2019-12-07T19:33:55.616+0000 7fe2d7285700 0 osd.0 18 crush map has features 288514119978713088 was 288514050185503233, adjusting msgr requires for mons 2019-12-07T19:33:55.616+0000 7fe2d7285700 0 osd.0 18 crush map has features 3314933069571702784, adjusting msgr requires for osds 2019-12-07T19:33:55.616+0000 7fe2d7285700 7 osd.0 18 consume_map version 18 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:1.identify_splits_and_merges identify_splits_and_merges slot 1.1s1 has no pg and waiting_for_split 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:2.identify_splits_and_merges identify_splits_and_merges slot 1.2s0 has no pg and waiting_for_split 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:5.identify_splits_and_merges identify_splits_and_merges slot 1.ds5 has no pg and waiting_for_split 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:7.identify_splits_and_merges identify_splits_and_merges slot 1.fs1 has no pg and waiting_for_split 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0 18 prune_pg_created 2019-12-07T19:33:55.616+0000 7fe2d7285700 10 osd.0:0.consume_map 18 (was 17) 2019-12-07T19:33:55.616+0000 7fe2d7285700 10 osd.0:1.consume_map 18 (was 17) 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.616+0000 7fe2d7285700 10 osd.0:2.consume_map 18 (was 17) 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.2s0 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.616+0000 7fe2d7285700 10 osd.0:3.consume_map 18 (was 17) 2019-12-07T19:33:55.616+0000 7fe2d7285700 10 osd.0:4.consume_map 18 (was 17) 2019-12-07T19:33:55.616+0000 7fe2d7285700 10 osd.0:5.consume_map 18 (was 17) 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:5._wake_pg_slot _wake_pg_slot 1.ds5 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7fe2d7285700 10 osd.0:6.consume_map 18 (was 17) 2019-12-07T19:33:55.616+0000 7fe2d7285700 10 osd.0:7.consume_map 18 (was 17) 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:33:55.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 pending_peering first epoch 18 <= 18, requeueing 2019-12-07T19:33:55.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.fs1 to_process <> waiting <> waiting_peering {18=} 2019-12-07T19:33:55.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.1s1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.1s1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.616+0000 7fe2c7265700 10 osd.0 18 handle_pg_query_nopg 1.1s1 2019-12-07T19:33:55.616+0000 7fe2c7265700 10 osd.0 18 pg 1.1s1 dne 2019-12-07T19:33:55.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 no pg, should create on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.616+0000 7fe2c7265700 10 osd.0 18 send_incremental_map 15 -> 18 to 0x558e818d7000 v2:172.21.15.131:6812/37076 2019-12-07T19:33:55.616+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c423972f:::inc_osdmap.16:0# 0x0~0 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1b9 size 0x1b9 (441) 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:55.616+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.ds5 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.616+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81867500 blob([0x25000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1b9) SharedBlob(0x558e81866070 sbid 0x0)) need 0x0~1b9 cache has 0x[] 2019-12-07T19:33:55.616+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.ds5 from 2(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81867500 blob([0x25000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1b9) SharedBlob(0x558e81866070 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1b9]} 2019-12-07T19:33:55.616+0000 7fe2c5261700 10 osd.0 18 handle_pg_query_nopg 1.ds5 2019-12-07T19:33:55.616+0000 7fe2c5261700 10 osd.0 18 pg 1.ds5 dne 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:55.616+0000 7fe2c5261700 10 osd.0 18 send_incremental_map 17 -> 18 to 0x558e818d6000 v2:172.21.15.131:6804/37077 2019-12-07T19:33:55.616+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:55.616+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.fs1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.616+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.616+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.fs1 from 3(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.616+0000 7fe2c425f700 10 osd.0 18 handle_pg_query_nopg 1.fs1 2019-12-07T19:33:55.616+0000 7fe2c425f700 10 osd.0 18 pg 1.fs1 dne 2019-12-07T19:33:55.616+0000 7fe2c425f700 10 osd.0 18 send_incremental_map 15 -> 18 to 0x558e818d7000 v2:172.21.15.131:6812/37076 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81867500 blob([0x25000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1b9) SharedBlob(0x558e81866070 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1b9]} 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e81866088 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e81171710 space 0x558e81866088 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:55.616+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c423972f:::inc_osdmap.16:0# 0x0~1b9 = 441 2019-12-07T19:33:55.616+0000 7fe2c7265700 10 osd.0 18 add_map_inc_bl 16 441 bytes 2019-12-07T19:33:55.616+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:cfa3972f:::inc_osdmap.17:0# 0x0~0 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~4f3 size 0x4f3 (1267) 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:55.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8000 blob([0x28000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4f3) SharedBlob(0x558e817e8070 sbid 0x0)) need 0x0~4f3 cache has 0x[0~4f3] 2019-12-07T19:33:55.616+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:cfa3972f:::inc_osdmap.17:0# 0x0~4f3 = 1267 2019-12-07T19:33:55.616+0000 7fe2c7265700 10 osd.0 18 add_map_inc_bl 17 1267 bytes 2019-12-07T19:33:55.616+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c15b972f:::inc_osdmap.18:0# 0x0~0 2019-12-07T19:33:55.617+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~259 size 0x259 (601) 2019-12-07T19:33:55.617+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:55.617+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81788230 blob([0x2b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x259) SharedBlob(0x558e817e83f0 sbid 0x0)) need 0x0~259 cache has 0x[0~259] 2019-12-07T19:33:55.617+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c15b972f:::inc_osdmap.18:0# 0x0~259 = 601 2019-12-07T19:33:55.617+0000 7fe2c7265700 10 osd.0 18 add_map_inc_bl 18 601 bytes 2019-12-07T19:33:55.617+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(16..18 src has 1..18) v4 -- 0x558e817de900 con 0x558e818d7000 2019-12-07T19:33:55.617+0000 7fe2c7265700 10 osd.0 18 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 15 -> 18 (shared) 2019-12-07T19:33:55.617+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 3 ==== pg_query2(1.9s3 1.9s3 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8191b900 con 0x558e818d7c00 2019-12-07T19:33:55.617+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify((query:18 sent:18 1.1s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558e80231c20 con 0x558e818d7000 2019-12-07T19:33:55.617+0000 7fe2e6d3a700 15 osd.0 18 enqueue_peering_evt 1.9s3 epoch_sent: 18 epoch_requested: 18 MQuery 1.9s3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.617+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.9s3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.617+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 2 ==== osd_map(16..18 src has 1..18) v4 ==== 2369+0+0 (unknown 0 0 0) 0x558e81908000 con 0x558e818d7c00 2019-12-07T19:33:55.617+0000 7fe2d4a80700 20 osd.0 18 OSD::ms_dispatch: osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.617+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:55.617+0000 7fe2c6a64700 10 osd.0 18 _make_pg 1.2s0 2019-12-07T19:33:55.617+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2(unlocked)] enter Initial 2019-12-07T19:33:55.617+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 4 ==== pg_query2(1.es4 1.es4 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8191b900 con 0x558e818d7c00 2019-12-07T19:33:55.617+0000 7fe2e6d3a700 15 osd.0 18 enqueue_peering_evt 1.es4 epoch_sent: 18 epoch_requested: 18 MQuery 1.es4 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.617+0000 7fe2e6d3a700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.es4 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.617+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.617+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.9s3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.617+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.617+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.9s3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.617+0000 7fe2cb26d700 10 osd.0 18 handle_pg_query_nopg 1.9s3 2019-12-07T19:33:55.617+0000 7fe2cb26d700 10 osd.0 18 pg 1.9s3 dne 2019-12-07T19:33:55.617+0000 7fe2cb26d700 10 osd.0 18 send_incremental_map 17 -> 18 to 0x558e818d7c00 v2:172.21.15.131:6823/37078 2019-12-07T19:33:55.617+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(18..18 src has 1..18) v4 -- 0x558e817df680 con 0x558e818d7c00 2019-12-07T19:33:55.617+0000 7fe2cb26d700 10 osd.0 18 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 17 -> 18 (shared) 2019-12-07T19:33:55.617+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:18 sent:18 1.9s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558e819b0000 con 0x558e818d7c00 2019-12-07T19:33:55.617+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:55.617+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.617+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.es4 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.617+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.617+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.es4 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.617+0000 7fe2c8a68700 10 osd.0 18 handle_pg_query_nopg 1.es4 2019-12-07T19:33:55.617+0000 7fe2c8a68700 10 osd.0 18 pg 1.es4 dne 2019-12-07T19:33:55.617+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:18 sent:18 1.es4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558e819b0b40 con 0x558e818d7c00 2019-12-07T19:33:55.617+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:55.617+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 1 ==== pg_query2(1.4s1 1.4s1 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8191b680 con 0x558e818fe000 2019-12-07T19:33:55.617+0000 7fe2e753b700 15 osd.0 18 enqueue_peering_evt 1.4s1 epoch_sent: 18 epoch_requested: 18 MQuery 1.4s1 from 7(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.617+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.4s1 from 7(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.4s1 from 7(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.4s1 from 7(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.617+0000 7fe2c9a6a700 10 osd.0 18 handle_pg_query_nopg 1.4s1 2019-12-07T19:33:55.617+0000 7fe2c9a6a700 10 osd.0 18 pg 1.4s1 dne 2019-12-07T19:33:55.617+0000 7fe2c9a6a700 10 osd.0 18 send_incremental_map 17 -> 18 to 0x558e818fe000 v2:172.21.15.109:6826/36939 2019-12-07T19:33:55.617+0000 7fe2c9a6a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(18..18 src has 1..18) v4 -- 0x558e817dfb00 con 0x558e818fe000 2019-12-07T19:33:55.617+0000 7fe2c9a6a700 10 osd.0 18 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 17 -> 18 (shared) 2019-12-07T19:33:55.617+0000 7fe2c9a6a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_notify((query:18 sent:18 1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558e819b10e0 con 0x558e818fe000 2019-12-07T19:33:55.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:55.617+0000 7fe2e2ca2700 10 osd.0 18 tick_without_osd_lock 2019-12-07T19:33:55.617+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffd2000/0x40000000/0x1680000000, data 0xda65/0x2c000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:55.617+0000 7fe2e2ca2700 25 osd.0 18 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153258+0000 last_rx_front 2019-12-07T19:33:51.153336+0000 2019-12-07T19:33:55.617+0000 7fe2e2ca2700 25 osd.0 18 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153377+0000 last_rx_front 2019-12-07T19:33:51.153481+0000 2019-12-07T19:33:55.617+0000 7fe2e2ca2700 25 osd.0 18 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153242+0000 last_rx_front 2019-12-07T19:33:51.153313+0000 2019-12-07T19:33:55.617+0000 7fe2e2ca2700 25 osd.0 18 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153637+0000 last_rx_front 2019-12-07T19:33:51.153527+0000 2019-12-07T19:33:55.618+0000 7fe2e2ca2700 25 osd.0 18 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153392+0000 last_rx_front 2019-12-07T19:33:51.153509+0000 2019-12-07T19:33:55.618+0000 7fe2e2ca2700 25 osd.0 18 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153611+0000 last_rx_front 2019-12-07T19:33:51.153694+0000 2019-12-07T19:33:55.618+0000 7fe2e2ca2700 25 osd.0 18 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:51.153000+0000 last_rx_back 2019-12-07T19:33:51.153670+0000 last_rx_front 2019-12-07T19:33:51.153496+0000 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 pg_epoch: 18 pg[1.2(unlocked)] enter NotTrimming 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach 1.2s0_head fresh osr 0x558e811de8c0 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 0 up [0,4,1,6,5,3] acting [0,4,1,6,5,3] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] init_hb_stamps now [0x558e811d0af0,0x558e811d01c0,0x558e811d0460,0x558e811d18f0,0x558e811d1570] 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 19.073583982s + 0s) 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:55.618+0000 7fe2c6a64700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:55.618+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:55.618+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:55.618949+0000 0 0.000000 2019-12-07T19:33:55.618+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=0 crt=0'0 mlcod 0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] activate_map 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] update_heartbeat_peers -> 0,1,3,4,5,6 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 18 need_heartbeat_peer_update 2019-12-07T19:33:55.618+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:55.618984+0000 1 0.000026 2019-12-07T19:33:55.618+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started 2019-12-07T19:33:55.618+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] enter Start 2019-12-07T19:33:55.618+0000 7fe2c6a64700 1 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] state: transitioning to Primary 2019-12-07T19:33:55.618+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] exit Start 2019-12-07T19:33:55.618996+0000 0 0.000000 2019-12-07T19:33:55.618+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 unknown mbc={}] enter Started/Primary 2019-12-07T19:33:55.618+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] enter Started/Primary/Peering 2019-12-07T19:33:55.618+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] enter Started/Primary/Peering/GetInfo 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] build_prior all_probe 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] build_prior final: probe 0(0),1(2),3(5),4(1),5(4),6(3) down blocked_by {} 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] up_thru 0 < same_since 18, must notify monitor 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: no prior_set down osds, clearing prior_readable_until_ub 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.1(2) 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.3(5) 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.4(1) 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.5(4) 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: querying info from osd.6(3) 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:55.618+0000 7fe2c6a64700 15 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] publish_stats_to_osd 0:0 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:33:55.618+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.2s2 1.2s2 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x558e8191bb80 con 0x558e818d7c00 2019-12-07T19:33:55.618+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_query2(1.2s5 1.2s5 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x558e819b6000 con 0x558e818d7000 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 18 send_incremental_map 17 -> 18 to 0x558e818fe400 v2:172.21.15.109:6810/36938 2019-12-07T19:33:55.618+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(18..18 src has 1..18) v4 -- 0x558e819b2240 con 0x558e818fe400 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 18 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 17 -> 18 (shared) 2019-12-07T19:33:55.618+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_query2(1.2s1 1.2s1 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x558e819b6280 con 0x558e818fe400 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 18 send_incremental_map 15 -> 18 to 0x558e818d6800 v2:172.21.15.109:6802/36936 2019-12-07T19:33:55.618+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(16..18 src has 1..18) v4 -- 0x558e819b2480 con 0x558e818d6800 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 18 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 15 -> 18 (shared) 2019-12-07T19:33:55.618+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x558e819b6500 con 0x558e818d6800 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 18 send_incremental_map 15 -> 18 to 0x558e818ffc00 v2:172.21.15.109:6818/36937 2019-12-07T19:33:55.618+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- osd_map(16..18 src has 1..18) v4 -- 0x558e819b26c0 con 0x558e818ffc00 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 18 maybe_share_map con 0x558e818ffc00 v2:172.21.15.109:6818/36937 map epoch 15 -> 18 (shared) 2019-12-07T19:33:55.618+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- pg_query2(1.2s3 1.2s3 query(info 0'0 epoch_sent 18) e18/18) v1 -- 0x558e819b6780 con 0x558e818ffc00 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e819a1200 seq 1 2019-12-07T19:33:55.618+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.2s0_head bits 4 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.2s0_head bits 4 = 0 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:40000000::::head# key 0x8080000000000000014000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:33:55.618+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1065 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:55.618+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._infover' <- _infover 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:55.618+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:55.618+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes 0x558e819b6a00 shared_blobs 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e819b6b08) update 0#1:40000000::::head# 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e819b6b08) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:40000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[] released 0x[] 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 18 handle_pg_create_info new pg pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0:2._attach_pg 1.2s0 0x558e81075000 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 18 identify_splits_and_merges 1.2s0 e18 to e18 pg_nums {18=16} 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0:2._prime_splits 2019-12-07T19:33:55.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 NullEvt +create_info 2019-12-07T19:33:55.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:33:55.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:55.618+0000 7fe2c5261700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(18..18 src has 1..18) v4 -- 0x558e817ded80 con 0x558e818d6000 2019-12-07T19:33:55.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:33:55.618+0000 7fe2c5261700 10 osd.0 18 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 17 -> 18 (shared) 2019-12-07T19:33:55.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:55.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:55.618+0000 7fe2c5261700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify((query:18 sent:18 1.ds5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558e817b4d20 con 0x558e818d6000 2019-12-07T19:33:55.618+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:55.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x10c99~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:55.618+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x10c99 0x10c99~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:55.618+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:55.618+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xc99 2019-12-07T19:33:55.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:55.618+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc99 2019-12-07T19:33:55.618+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:55.618+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x206 and padding block with 0xdfa 2019-12-07T19:33:55.618+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(16..18 src has 1..18) v4 -- 0x558e817df200 con 0x558e818d7000 2019-12-07T19:33:55.618+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 3 ==== pg_notify((query:18 sent:18 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (unknown 0 0 0) 0x558e819b10e0 con 0x558e818d6800 2019-12-07T19:33:55.618+0000 7fe2e753b700 7 osd.0 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.5 2019-12-07T19:33:55.618+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify((query:18 sent:18 1.fs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558e817b52c0 con 0x558e818d7000 2019-12-07T19:33:55.618+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:55.618+0000 7fe2e753b700 15 osd.0 18 enqueue_peering_evt 1.2s0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 5(4) notify: (query:18 sent:18 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.618+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 5(4) notify: (query:18 sent:18 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.618+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11206 2019-12-07T19:33:55.618+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:55.618+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:55.618+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:55.618+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:55.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000108070s (0.000000140s flush + 0.000107930s kv commit) 2019-12-07T19:33:55.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:55.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:55.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1200> 2019-12-07T19:33:55.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:55.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:33:55.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:33:55.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:33:55.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 0x558e819b6a00 2019-12-07T19:33:55.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:33:55.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:33:55.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:33:55.618+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:55.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:55.618+0000 7fe2e2ca2700 20 osd.0 18 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:55.618+0000 7fe2d7285700 10 osd.0 18 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:55.618+0000 7fe2e2ca2700 20 osd.0 18 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 18 queue_want_up_thru now 18 (was 0), currently 0 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 18 send_alive up_thru currently 0 want 18 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 18 send_alive want 18 2019-12-07T19:33:55.618+0000 7fe2d7285700 15 osd.0 18 enqueue_peering_evt 1.2s0 epoch_sent: 18 epoch_requested: 18 NullEvt 2019-12-07T19:33:55.618+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt) prio 255 cost 10 e18) 2019-12-07T19:33:55.618+0000 7fe2d7285700 10 osd.0 18 maybe_update_heartbeat_peers updating 2019-12-07T19:33:55.618+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- osd_alive(want up_thru 18 have 18) v1 -- 0x558e819b14a0 con 0x558e80f39000 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 5(4) notify: (query:18 sent:18 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 5(4) notify: (query:18 sent:18 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x558e81075000 2019-12-07T19:33:55.618+0000 7fe2d7285700 10 osd.0 18 adding neighbor peer osd.1 2019-12-07T19:33:55.618+0000 7fe2d7285700 10 osd.0 18 adding neighbor peer osd.5 2019-12-07T19:33:55.618+0000 7fe2d7285700 10 osd.0 18 adding neighbor peer osd.7 2019-12-07T19:33:55.618+0000 7fe2d7285700 10 osd.0 18 adding random peer osd.3 2019-12-07T19:33:55.618+0000 7fe2d7285700 10 osd.0 18 adding random peer osd.4 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 5(4) notify: (query:18 sent:18 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.618+0000 7fe2d7285700 10 osd.0 18 adding random peer osd.6 2019-12-07T19:33:55.618+0000 7fe2d7285700 10 osd.0 18 maybe_update_heartbeat_peers 7 peers, extras 1,2,3,4,5,6,7 2019-12-07T19:33:55.618+0000 7fe2d7285700 7 osd.0 18 activate_map version 18 2019-12-07T19:33:55.618+0000 7fe2cc26f700 20 osd.0 18 agent_entry empty queue 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.5(4) 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.618+0000 7fe2e2ca2700 20 osd.0 18 scrub_load_below_threshold loadavg per cpu 0.07 < max 5 = yes 2019-12-07T19:33:55.618+0000 7fe2e2ca2700 20 osd.0 18 sched_scrub load_is_low=1 2019-12-07T19:33:55.618+0000 7fe2e2ca2700 10 osd.0 18 sched_scrub 1.2s0 scheduled at 2019-12-07T19:35:01.133068+0000 > 2019-12-07T19:33:55.619520+0000 2019-12-07T19:33:55.618+0000 7fe2e2ca2700 20 osd.0 18 sched_scrub done 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:01.133068+0000, must = 0 2019-12-07T19:33:55.618+0000 7fe2e2ca2700 10 osd.0 18 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:55.618+0000 7fe2e2ca2700 20 osd.0 18 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:55.618+0000 7fe2e2ca2700 10 osd.0 18 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 0,1,3,4,5,6 unchanged 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 NullEvt) prio 255 cost 10 e18) pg 0x558e81075000 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 NullEvt 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:33:55.618+0000 7fe2c6a64700 10 osd.0 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.618+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:55.618+0000 7fe2d4a80700 10 osd.0 18 do_waiters -- start 2019-12-07T19:33:55.618+0000 7fe2d4a80700 10 osd.0 18 do_waiters -- finish 2019-12-07T19:33:55.618+0000 7fe2d4a80700 20 osd.0 18 _dispatch 0x558e81908000 osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.618+0000 7fe2d4a80700 3 osd.0 18 handle_osd_map epochs [16,18], i have 18, src has [1,18] 2019-12-07T19:33:55.618+0000 7fe2d4a80700 10 osd.0 18 no new maps here, dropping 2019-12-07T19:33:55.618+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 5 ==== osd_map(16..18 src has 1..18) v4 ==== 2369+0+0 (unknown 0 0 0) 0x558e819086c0 con 0x558e818d7c00 2019-12-07T19:33:55.618+0000 7fe2d4a80700 20 osd.0 18 OSD::ms_dispatch: osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.618+0000 7fe2d4a80700 10 osd.0 18 do_waiters -- start 2019-12-07T19:33:55.618+0000 7fe2d4a80700 10 osd.0 18 do_waiters -- finish 2019-12-07T19:33:55.618+0000 7fe2d4a80700 20 osd.0 18 _dispatch 0x558e819086c0 osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.618+0000 7fe2d4a80700 3 osd.0 18 handle_osd_map epochs [16,18], i have 18, src has [1,18] 2019-12-07T19:33:55.618+0000 7fe2d4a80700 10 osd.0 18 no new maps here, dropping 2019-12-07T19:33:55.618+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 2 ==== osd_map(18..18 src has 1..18) v4 ==== 645+0+0 (unknown 0 0 0) 0x558e817de240 con 0x558e818fe000 2019-12-07T19:33:55.619+0000 7fe2d4a80700 20 osd.0 18 OSD::ms_dispatch: osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.619+0000 7fe2d4a80700 10 osd.0 18 do_waiters -- start 2019-12-07T19:33:55.619+0000 7fe2d4a80700 10 osd.0 18 do_waiters -- finish 2019-12-07T19:33:55.619+0000 7fe2d4a80700 20 osd.0 18 _dispatch 0x558e817de240 osd_map(18..18 src has 1..18) v4 2019-12-07T19:33:55.619+0000 7fe2d4a80700 3 osd.0 18 handle_osd_map epochs [18,18], i have 18, src has [1,18] 2019-12-07T19:33:55.619+0000 7fe2d4a80700 10 osd.0 18 no new maps here, dropping 2019-12-07T19:33:55.618+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 4 ==== pg_notify((query:18 sent:18 1.2s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (unknown 0 0 0) 0x558e80231c20 con 0x558e818d7000 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 7 osd.0 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.2s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.3 2019-12-07T19:33:55.619+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 2 ==== osd_map(16..18 src has 1..18) v4 ==== 2369+0+0 (unknown 0 0 0) 0x558e819b2480 con 0x558e818d6800 2019-12-07T19:33:55.619+0000 7fe2d4a80700 20 osd.0 18 OSD::ms_dispatch: osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.619+0000 7fe2d4a80700 10 osd.0 18 do_waiters -- start 2019-12-07T19:33:55.619+0000 7fe2d4a80700 10 osd.0 18 do_waiters -- finish 2019-12-07T19:33:55.619+0000 7fe2d4a80700 20 osd.0 18 _dispatch 0x558e819b2480 osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.619+0000 7fe2d4a80700 3 osd.0 18 handle_osd_map epochs [16,18], i have 18, src has [1,18] 2019-12-07T19:33:55.619+0000 7fe2d4a80700 10 osd.0 18 no new maps here, dropping 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 15 osd.0 18 enqueue_peering_evt 1.2s0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 3(5) notify: (query:18 sent:18 1.2s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 3(5) notify: (query:18 sent:18 1.2s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 3(5) notify: (query:18 sent:18 1.2s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 3(5) notify: (query:18 sent:18 1.2s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x558e81075000 2019-12-07T19:33:55.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:55.619+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 3(5) notify: (query:18 sent:18 1.2s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.619+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.3(5) 1.2s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.619+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:20.849750+0000, must = 0 2019-12-07T19:33:55.619+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 0,1,3,4,5,6 unchanged 2019-12-07T19:33:55.619+0000 7fe2caa6c700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 3 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.619+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:55.619+0000 7fe2caa6c700 10 osd.0 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 6 ==== pg_query2(1.7s3 1.7s3 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e819b6000 con 0x558e818d7c00 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 15 osd.0 18 enqueue_peering_evt 1.7s3 epoch_sent: 18 epoch_requested: 18 MQuery 1.7s3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.7s3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 7 ==== pg_query2(1.as3 1.as3 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8191bb80 con 0x558e818d7c00 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 15 osd.0 18 enqueue_peering_evt 1.as3 epoch_sent: 18 epoch_requested: 18 MQuery 1.as3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.as3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.619+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.7s3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.619+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.7s3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.619+0000 7fe2c8267700 10 osd.0 18 handle_pg_query_nopg 1.7s3 2019-12-07T19:33:55.619+0000 7fe2c8267700 10 osd.0 18 pg 1.7s3 dne 2019-12-07T19:33:55.619+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:18 sent:18 1.7s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558e819b1c20 con 0x558e818d7c00 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 8 ==== pg_notify((query:18 sent:18 1.2s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (unknown 0 0 0) 0x558e819b0b40 con 0x558e818d7c00 2019-12-07T19:33:55.619+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 7 osd.0 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.2s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.1 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 15 osd.0 18 enqueue_peering_evt 1.2s0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 1(2) notify: (query:18 sent:18 1.2s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 1(2) notify: (query:18 sent:18 1.2s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.as3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.as3 from 1(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.619+0000 7fe2c6a64700 10 osd.0 18 handle_pg_query_nopg 1.as3 2019-12-07T19:33:55.619+0000 7fe2c6a64700 10 osd.0 18 pg 1.as3 dne 2019-12-07T19:33:55.619+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:18 sent:18 1.as3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558e819c4000 con 0x558e818d7c00 2019-12-07T19:33:55.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 1(2) notify: (query:18 sent:18 1.2s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 1(2) notify: (query:18 sent:18 1.2s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x558e81075000 2019-12-07T19:33:55.619+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 1(2) notify: (query:18 sent:18 1.2s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.619+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.1(2) 1.2s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.619+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:24.209557+0000, must = 0 2019-12-07T19:33:55.619+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 0,1,3,4,5,6 unchanged 2019-12-07T19:33:55.619+0000 7fe2c6a64700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 1 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.619+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:33:55.619+0000 7fe2c6a64700 10 osd.0 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 3 ==== pg_notify((query:18 sent:18 1.2s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (unknown 0 0 0) 0x558e819b0b40 con 0x558e818fe400 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 7 osd.0 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.2s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.4 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 15 osd.0 18 enqueue_peering_evt 1.2s0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 4(1) notify: (query:18 sent:18 1.2s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.619+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 4(1) notify: (query:18 sent:18 1.2s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 4(1) notify: (query:18 sent:18 1.2s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 4(1) notify: (query:18 sent:18 1.2s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x558e81075000 2019-12-07T19:33:55.619+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 4(1) notify: (query:18 sent:18 1.2s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.619+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 2 ==== osd_map(16..18 src has 1..18) v4 ==== 2369+0+0 (unknown 0 0 0) 0x558e817df680 con 0x558e818fe400 2019-12-07T19:33:55.619+0000 7fe2d4a80700 20 osd.0 18 OSD::ms_dispatch: osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.619+0000 7fe2d4a80700 10 osd.0 18 do_waiters -- start 2019-12-07T19:33:55.619+0000 7fe2d4a80700 10 osd.0 18 do_waiters -- finish 2019-12-07T19:33:55.619+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.4(1) 1.2s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.619+0000 7fe2d4a80700 20 osd.0 18 _dispatch 0x558e817df680 osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.619+0000 7fe2d4a80700 3 osd.0 18 handle_osd_map epochs [16,18], i have 18, src has [1,18] 2019-12-07T19:33:55.619+0000 7fe2d4a80700 10 osd.0 18 no new maps here, dropping 2019-12-07T19:33:55.619+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:18.776912+0000, must = 0 2019-12-07T19:33:55.619+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 0,1,3,4,5,6 unchanged 2019-12-07T19:33:55.619+0000 7fe2caa6c700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 4 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.619+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:55.619+0000 7fe2caa6c700 10 osd.0 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:55.621+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 4 ==== pg_query2(1.bs3 1.bs3 query(info 0'0 epoch_sent 18) e18/18) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8191bb80 con 0x558e818fe400 2019-12-07T19:33:55.621+0000 7fe2e6d3a700 15 osd.0 18 enqueue_peering_evt 1.bs3 epoch_sent: 18 epoch_requested: 18 MQuery 1.bs3 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18) 2019-12-07T19:33:55.621+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.bs3 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) 2019-12-07T19:33:55.621+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.621+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.bs3 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.621+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:55.621+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MQuery 1.bs3 from 4(0) query_epoch 18 query: query(info 0'0 epoch_sent 18)) prio 255 cost 10 e18) pg 0 2019-12-07T19:33:55.621+0000 7fe2ca26b700 10 osd.0 18 handle_pg_query_nopg 1.bs3 2019-12-07T19:33:55.621+0000 7fe2ca26b700 10 osd.0 18 pg 1.bs3 dne 2019-12-07T19:33:55.621+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify((query:18 sent:18 1.bs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 18) v7 -- 0x558e817b52c0 con 0x558e818fe400 2019-12-07T19:33:55.621+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:55.624+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 2 ==== pg_notify((query:18 sent:18 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 18) v7 ==== 985+0+0 (unknown 0 0 0) 0x558e819b14a0 con 0x558e818ffc00 2019-12-07T19:33:55.624+0000 7fe2e7d3c700 7 osd.0 18 handle_fast_pg_notify pg_notify((query:18 sent:18 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 18) v7 from osd.6 2019-12-07T19:33:55.624+0000 7fe2e7d3c700 15 osd.0 18 enqueue_peering_evt 1.2s0 epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 6(3) notify: (query:18 sent:18 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.624+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 6(3) notify: (query:18 sent:18 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 6(3) notify: (query:18 sent:18 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) queued 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18)> waiting <> waiting_peering {} 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 6(3) notify: (query:18 sent:18 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e18) pg 0x558e81075000 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] do_peering_event: epoch_sent: 18 epoch_requested: 18 MNotifyRec 1.2s0 from 6(3) notify: (query:18 sent:18 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] got osd.6(3) 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.624+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 3 ==== osd_map(16..18 src has 1..18) v4 ==== 2369+0+0 (unknown 0 0 0) 0x558e819b26c0 con 0x558e818ffc00 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:23.718605+0000, must = 0 2019-12-07T19:33:55.624+0000 7fe2d4a80700 20 osd.0 18 OSD::ms_dispatch: osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.624+0000 7fe2d4a80700 10 osd.0 18 do_waiters -- start 2019-12-07T19:33:55.624+0000 7fe2d4a80700 10 osd.0 18 do_waiters -- finish 2019-12-07T19:33:55.624+0000 7fe2d4a80700 20 osd.0 18 _dispatch 0x558e819b26c0 osd_map(16..18 src has 1..18) v4 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] update_heartbeat_peers 0,1,3,4,5,6 unchanged 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Adding osd: 6 peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.624+0000 7fe2d4a80700 3 osd.0 18 handle_osd_map epochs [16,18], i have 18, src has [1,18] 2019-12-07T19:33:55.624+0000 7fe2d4a80700 10 osd.0 18 no new maps here, dropping 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Common peer features: 3f01cfb8ffadffff 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Common acting features: 3f01cfb8ffadffff 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Common upacting features: 3f01cfb8ffadffff 2019-12-07T19:33:55.624+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering/GetInfo 2019-12-07T19:33:55.624955+0000 7 0.000290 2019-12-07T19:33:55.624+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] enter Started/Primary/Peering/GetLog 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.0(0) 1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.1(2) 1.2s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.3(5) 1.2s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.4(1) 1.2s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.5(4) 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting all_info osd.6(3) 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] For position 0: selecting up[i]: 0(0) For position 1: selecting up[i]: 4(1) For position 2: selecting up[i]: 1(2) For position 3: selecting up[i]: 6(3) For position 4: selecting up[i]: 5(4) For position 5: selecting up[i]: 3(5) 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_async_recovery_ec candidates by cost are: 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_async_recovery_ec result want=[0,4,1,6,5,3] async_recovery= 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] acting_recovery_backfill is 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] choose_acting want=[0,4,1,6,5,3] backfill_targets= async_recovery_targets= 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: leaving GetLog 2019-12-07T19:33:55.624+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering/GetLog 2019-12-07T19:33:55.625109+0000 0 0.000000 2019-12-07T19:33:55.624+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] enter Started/Primary/Peering/GetMissing 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: still need up_thru update before going active 2019-12-07T19:33:55.624+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering/GetMissing 2019-12-07T19:33:55.625134+0000 0 0.000000 2019-12-07T19:33:55.624+0000 7fe2c6a64700 5 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] enter Started/Primary/Peering/WaitUpThru 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e819a1500 seq 2 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1500 cost 4032 (1 ios * 4000 + 32 bytes) 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1500 onodes shared_blobs 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1500 allocated 0x[] released 0x[] 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 prepare 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1500 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 io_done 2019-12-07T19:33:55.624+0000 7fe2c6a64700 10 osd.0 18 queue_want_up_thru want 18 <= queued 18, currently 0 2019-12-07T19:33:55.624+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:55.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:55.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:55.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:33:55.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:33:55.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:33:55.624+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:55.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11206~2e to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:55.624+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11206 0x11206~2e to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:55.624+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:55.624+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x1206 2019-12-07T19:33:55.624+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x206 2019-12-07T19:33:55.624+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:55.624+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x234 and padding block with 0xdcc 2019-12-07T19:33:55.624+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11234 2019-12-07T19:33:55.624+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:55.624+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:55.624+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:55.624+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:55.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000133399s (0.000000093s flush + 0.000133306s kv commit) 2019-12-07T19:33:55.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:55.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:55.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1500> 2019-12-07T19:33:55.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:55.624+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 kv_submitted 2019-12-07T19:33:55.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1500 2019-12-07T19:33:55.624+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 finishing 2019-12-07T19:33:55.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1500 onodes 2019-12-07T19:33:55.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1500 done 2019-12-07T19:33:55.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:33:55.624+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1500 [] 2019-12-07T19:33:55.624+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:55.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:55.625+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:55.634+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.684+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.732+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e81985c00 0x558e81988100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:55.732+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e81985c00 0x558e81988100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:55.733+0000 7fe2e7d3c700 10 osd.0 18 ms_handle_authentication new session 0x558e818c3ba0 con 0x558e81985c00 entity client.admin addr 2019-12-07T19:33:55.733+0000 7fe2e7d3c700 10 osd.0 18 ms_handle_authentication session 0x558e818c3ba0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:55.733+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/808367751 conn(0x558e81985c00 0x558e81988100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=client.4528 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:55.733+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4528 172.21.15.131:0/808367751 1 ==== command(tid 1: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558e819b14a0 con 0x558e81985c00 2019-12-07T19:33:55.733+0000 7fe2d959f700 20 osd.0 18 OSD::ms_dispatch: command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:55.733+0000 7fe2d959f700 10 osd.0 18 do_waiters -- start 2019-12-07T19:33:55.733+0000 7fe2d959f700 10 osd.0 18 do_waiters -- finish 2019-12-07T19:33:55.733+0000 7fe2d959f700 20 osd.0 18 _dispatch 0x558e819b14a0 command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:55.733+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/808367751 -- command_reply(tid 1: 0 ) v1 -- 0x558e8022f180 con 0x558e81985c00 2019-12-07T19:33:55.735+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.737+0000 7fe2e4535700 10 osd.0 18 tick 2019-12-07T19:33:55.737+0000 7fe2e4535700 10 osd.0 18 do_waiters -- start 2019-12-07T19:33:55.737+0000 7fe2e4535700 10 osd.0 18 do_waiters -- finish 2019-12-07T19:33:55.737+0000 7fe2e4535700 20 osd.0 18 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:55.756+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4528 172.21.15.131:0/808367751 2 ==== command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558e817b41e0 con 0x558e81985c00 2019-12-07T19:33:55.756+0000 7fe2d959f700 20 osd.0 18 OSD::ms_dispatch: command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:33:55.756+0000 7fe2d959f700 10 osd.0 18 do_waiters -- start 2019-12-07T19:33:55.756+0000 7fe2d959f700 10 osd.0 18 do_waiters -- finish 2019-12-07T19:33:55.756+0000 7fe2d959f700 20 osd.0 18 _dispatch 0x558e817b41e0 command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:33:55.758+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/808367751 -- command_reply(tid 2: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558e8022ee00 con 0x558e81985c00 2019-12-07T19:33:55.785+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.802+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:33:55.835+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.885+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.935+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:55.943+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/808367751 conn(0x558e81985c00 msgr2=0x558e81988100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 83 2019-12-07T19:33:55.943+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/808367751 conn(0x558e81985c00 msgr2=0x558e81988100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:55.943+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/808367751 conn(0x558e81985c00 0x558e81988100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:55.943+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/808367751 conn(0x558e81985c00 0x558e81988100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:55.943+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] reap_dead start 2019-12-07T19:33:55.943+0000 7fe2d959f700 2 osd.0 18 ms_handle_reset con 0x558e81985c00 session 0x558e818c3ba0 2019-12-07T19:33:55.953+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.6 172.21.15.109:0/36937 4 ==== osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 0.000042793s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8800 con 0x558e802a4c00 2019-12-07T19:33:55.953+0000 7fe2e753b700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709119190s,0.000042793s]) 2019-12-07T19:33:55.953+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.409180863s delta_ub 0.000432426s) v5 -- 0x558e80264000 con 0x558e802a4c00 2019-12-07T19:33:55.953+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.6 172.21.15.109:0/36937 4 ==== osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.408748437s delta_ub 0.000042793s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8800 con 0x558e8182cc00 2019-12-07T19:33:55.953+0000 7fe2e753b700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709078998s,0.000042793s]) 2019-12-07T19:33:55.953+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:55.954243+0000/19.408748437s send_stamp 19.409221055s delta_ub 0.000472618s) v5 -- 0x558e811f7a00 con 0x558e8182cc00 2019-12-07T19:33:55.985+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.035+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.085+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.135+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.185+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.205+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:33:56.235+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.286+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.336+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.342+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.5 172.21.15.109:0/36936 6 ==== osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.342664+0000/19.808987388s send_stamp 19.808987388s delta_ub 0.011977110s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118f400 con 0x558e8182c800 2019-12-07T19:33:56.342+0000 7fe2e7d3c700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011296042s,0.011977110s]) 2019-12-07T19:33:56.342+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.342664+0000/19.808987388s send_stamp 19.797691346s delta_ub 18446744073.698255574s) v5 -- 0x558e811f7800 con 0x558e8182c800 2019-12-07T19:33:56.342+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.5 172.21.15.109:0/36936 6 ==== osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.342664+0000/19.808987388s send_stamp 19.808987388s delta_ub 0.011977110s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118f400 con 0x558e8181b800 2019-12-07T19:33:56.342+0000 7fe2e7d3c700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011228439s,0.011977110s]) 2019-12-07T19:33:56.342+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.342664+0000/19.808987388s send_stamp 19.797758949s delta_ub 18446744073.698323177s) v5 -- 0x558e818eee00 con 0x558e8181b800 2019-12-07T19:33:56.386+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.436+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 5 osd.0 18 heartbeat osd_stat(store_statfs(0x163ffd2000/0x40000000/0x1680000000, data 0xda65/0x2c000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5,6,7] op hist []) 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 20 osd.0 18 check_full_status cur ratio 1.94973e-06, physical ratio 1.94973e-06, new state none 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 18446744073.707560937s) v5 -- 0x558e811f6200 con 0x558e818d7800 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 18446744073.707560937s) v5 -- 0x558e811f7200 con 0x558e818d7400 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 18446744073.692716198s) v5 -- 0x558e811f7000 con 0x558e817ff000 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 18446744073.692716198s) v5 -- 0x558e811f6c00 con 0x558e817ff400 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 18446744073.706874541s) v5 -- 0x558e811f6400 con 0x558e817ff800 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 18446744073.706874541s) v5 -- 0x558e811f6600 con 0x558e817ffc00 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 0.005855705s) v5 -- 0x558e811f6e00 con 0x558e8181a000 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 0.005855705s) v5 -- 0x558e818a8e00 con 0x558e8181a400 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 18446744073.698323177s) v5 -- 0x558e811f7400 con 0x558e817fe000 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 18446744073.698323177s) v5 -- 0x558e811f7200 con 0x558e817fe400 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 0.000472618s) v5 -- 0x558e818ef400 con 0x558e8181a800 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 0.000472618s) v5 -- 0x558e818ef000 con 0x558e8181ac00 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 0.019543146s) v5 -- 0x558e818ee200 con 0x558e817fe800 2019-12-07T19:33:56.452+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908022868s delta_ub 0.019543146s) v5 -- 0x558e818ee800 con 0x558e817fec00 2019-12-07T19:33:56.452+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6808/37077 5 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.925200616s delta_ub 0.017177748s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e817ff000 2019-12-07T19:33:56.452+0000 7fe2e753b700 25 osd.0 18 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:51.153377+0000 -> 2019-12-07T19:33:56.453559+0000 last_rx_front 2019-12-07T19:33:51.153481+0000 2019-12-07T19:33:56.452+0000 7fe2e753b700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016949585s,0.017177748s]) 2019-12-07T19:33:56.452+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6816/37076 5 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.911016712s delta_ub 0.002993844s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e811f6800 con 0x558e817ff800 2019-12-07T19:33:56.452+0000 7fe2e7d3c700 25 osd.0 18 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:51.153242+0000 -> 2019-12-07T19:33:56.453575+0000 last_rx_front 2019-12-07T19:33:51.153313+0000 2019-12-07T19:33:56.452+0000 7fe2e7d3c700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002749128s,0.002993844s]) 2019-12-07T19:33:56.452+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6814/37076 5 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.911051829s delta_ub 0.003028961s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e817ffc00 2019-12-07T19:33:56.452+0000 7fe2e753b700 25 osd.0 18 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453575+0000 last_rx_front 2019-12-07T19:33:51.153313+0000 -> 2019-12-07T19:33:56.453596+0000 2019-12-07T19:33:56.452+0000 7fe2e753b700 25 osd.0 18 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:33:56.453331+0000) and older pending ping(s) 2019-12-07T19:33:56.452+0000 7fe2e753b700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002764194s,0.003028961s]) 2019-12-07T19:33:56.452+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6806/37077 5 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.925280012s delta_ub 0.017257144s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8200 con 0x558e817ff400 2019-12-07T19:33:56.452+0000 7fe2e6d3a700 25 osd.0 18 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453559+0000 last_rx_front 2019-12-07T19:33:51.153481+0000 -> 2019-12-07T19:33:56.453627+0000 2019-12-07T19:33:56.452+0000 7fe2e6d3a700 25 osd.0 18 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:33:56.453331+0000) and older pending ping(s) 2019-12-07T19:33:56.452+0000 7fe2e6d3a700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016960406s,0.017257144s]) 2019-12-07T19:33:56.452+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6831/37078 1 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.910418599s delta_ub 0.002395731s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e818d7800 2019-12-07T19:33:56.453+0000 7fe2e753b700 25 osd.0 18 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:51.153258+0000 -> 2019-12-07T19:33:56.453695+0000 last_rx_front 2019-12-07T19:33:51.153336+0000 2019-12-07T19:33:56.453+0000 7fe2e753b700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002032004s,0.002395731s]) 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6827/37078 1 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.910562112s delta_ub 0.002539244s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8200 con 0x558e818d7400 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 25 osd.0 18 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453695+0000 last_rx_front 2019-12-07T19:33:51.153336+0000 -> 2019-12-07T19:33:56.453704+0000 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 25 osd.0 18 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:33:56.453331+0000) and older pending ping(s) 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002166438s,0.002539244s]) 2019-12-07T19:33:56.453+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6828/36939 5 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.889020596s delta_ub 18446744073.690549344s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e811f6800 con 0x558e817fec00 2019-12-07T19:33:56.453+0000 7fe2e7d3c700 25 osd.0 18 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:51.153670+0000 last_rx_front 2019-12-07T19:33:51.153496+0000 -> 2019-12-07T19:33:56.453733+0000 2019-12-07T19:33:56.453+0000 7fe2e7d3c700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690147600s,18446744073.690549344s]) 2019-12-07T19:33:56.453+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.6 v2:172.21.15.109:6822/36937 5 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908085322s delta_ub 0.000062454s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e8181a800 2019-12-07T19:33:56.453+0000 7fe2e753b700 25 osd.0 18 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:51.153611+0000 -> 2019-12-07T19:33:56.453751+0000 last_rx_front 2019-12-07T19:33:51.153694+0000 2019-12-07T19:33:56.453+0000 7fe2e753b700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709193740s,0.000062454s]) 2019-12-07T19:33:56.453+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6806/36936 5 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.919901272s delta_ub 0.011878404s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e811f6800 con 0x558e817fe000 2019-12-07T19:33:56.453+0000 7fe2e7d3c700 25 osd.0 18 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:51.153392+0000 -> 2019-12-07T19:33:56.453763+0000 last_rx_front 2019-12-07T19:33:51.153509+0000 2019-12-07T19:33:56.453+0000 7fe2e7d3c700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011445917s,0.011878404s]) 2019-12-07T19:33:56.453+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6804/36936 5 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.919936668s delta_ub 0.011913800s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e817fe400 2019-12-07T19:33:56.453+0000 7fe2e753b700 25 osd.0 18 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453763+0000 last_rx_front 2019-12-07T19:33:51.153509+0000 -> 2019-12-07T19:33:56.453773+0000 2019-12-07T19:33:56.453+0000 7fe2e753b700 25 osd.0 18 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:33:56.453331+0000) and older pending ping(s) 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6830/36939 5 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.889071669s delta_ub 18446744073.690600417s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8200 con 0x558e817fe800 2019-12-07T19:33:56.453+0000 7fe2e753b700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011471266s,0.011913800s]) 2019-12-07T19:33:56.453+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6812/36938 5 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.902849414s delta_ub 18446744073.704378162s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e811f6800 con 0x558e8181a400 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 25 osd.0 18 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:51.153670+0000 -> 2019-12-07T19:33:56.453789+0000 last_rx_front 2019-12-07T19:33:56.453733+0000 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 25 osd.0 18 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:33:56.453331+0000) and older pending ping(s) 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690142012s,18446744073.690600417s]) 2019-12-07T19:33:56.453+0000 7fe2e7d3c700 25 osd.0 18 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:51.153637+0000 last_rx_front 2019-12-07T19:33:51.153527+0000 -> 2019-12-07T19:33:56.453807+0000 2019-12-07T19:33:56.453+0000 7fe2e7d3c700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703901978s,18446744073.704378162s]) 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.6 v2:172.21.15.109:6820/36937 5 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.908122228s delta_ub 0.000099360s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8200 con 0x558e8181ac00 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 25 osd.0 18 handle_osd_ping got reply from osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453751+0000 last_rx_front 2019-12-07T19:33:51.153694+0000 -> 2019-12-07T19:33:56.453819+0000 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 25 osd.0 18 handle_osd_ping got all replies from osd.6 , erase pending ping(sent at 2019-12-07T19:33:56.453331+0000) and older pending ping(s) 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709162457s,0.000099360s]) 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6814/36938 5 ==== osd_ping(ping_reply e18 up_from 13 ping_stamp 2019-12-07T19:33:56.453331+0000/19.908022868s send_stamp 19.902806678s delta_ub 18446744073.704335426s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8200 con 0x558e8181a000 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 25 osd.0 18 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:51.153637+0000 -> 2019-12-07T19:33:56.453839+0000 last_rx_front 2019-12-07T19:33:56.453807+0000 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 25 osd.0 18 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:33:56.453331+0000) and older pending ping(s) 2019-12-07T19:33:56.453+0000 7fe2e6d3a700 20 osd.0 18 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703827048s,18446744073.704335426s]) 2019-12-07T19:33:56.486+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.536+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.586+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:33:56.586+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:33:56.586+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:33:56.586+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:33:56.586+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:33:56.586+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:33:56.586+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:33:56.586+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:33:56.586+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:33:56.586+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:33:56.586+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:33:56.586+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:33:56.586+0000 7fe2dc28f700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:33:56.586+0000 7fe2dc28f700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258776 2019-12-07T19:33:56.586+0000 7fe2dc28f700 5 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 50426 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.613+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 5 ==== PGlog(1.bs3 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x558e80e53000 con 0x558e818fe400 2019-12-07T19:33:56.613+0000 7fe2e6d3a700 15 osd.0 18 enqueue_peering_evt 1.bs3 epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info 2019-12-07T19:33:56.613+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 map 19 > 18 2019-12-07T19:33:56.613+0000 7fe2c6263700 20 osd.0 op_wq _add_slot_waiter 1.bs3 peering, item epoch is 19, will wait on OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.615+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 15 ==== pg_create2(e19 {1.2s0=18,2019-12-07T19:33:55.603918+0000}) v2 ==== 207+0+0 (crc 0 0 0) 0x558e819098c0 con 0x558e80f39000 2019-12-07T19:33:56.615+0000 7fe2e7d3c700 7 osd.0 18 handle_fast_pg_create pg_create2(e19 {1.2s0=18,2019-12-07T19:33:55.603918+0000}) v2 from mon.2 2019-12-07T19:33:56.615+0000 7fe2e7d3c700 20 osd.0 18 handle_fast_pg_create 1.2s0 e18@2019-12-07T19:33:55.603918+0000 history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 pi ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.615+0000 7fe2e7d3c700 15 osd.0 18 enqueue_peering_evt 1.2s0 epoch_sent: 19 epoch_requested: 19 NullEvt +create_info 2019-12-07T19:33:56.615+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.615+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.615+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.615+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.615+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt +create_info) prio 255 cost 10 e19) pg 0x558e81075000 2019-12-07T19:33:56.615+0000 7fe2c6a64700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 19, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.615+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.616+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 16 ==== osd_map(18..19 src has 1..19) v4 ==== 1298+0+0 (crc 0 0 0) 0x558e819098c0 con 0x558e80f39000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 osd.0 18 OSD::ms_dispatch: osd_map(18..19 src has 1..19) v4 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 osd.0 18 do_waiters -- start 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 osd.0 18 do_waiters -- finish 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 osd.0 18 _dispatch 0x558e819098c0 osd_map(18..19 src has 1..19) v4 2019-12-07T19:33:56.616+0000 7fe2d959f700 3 osd.0 18 handle_osd_map epochs [18,19], i have 18, src has [1,19] 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 osd.0 18 handle_osd_map got inc map for epoch 19 2019-12-07T19:33:56.616+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 5 ==== PGlog(1.fs1 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x558e811b8000 con 0x558e818d7000 2019-12-07T19:33:56.616+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:39c0e826:::osdmap.18:0# 0x0~0 2019-12-07T19:33:56.616+0000 7fe2e6d3a700 15 osd.0 18 enqueue_peering_evt 1.fs1 epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info 2019-12-07T19:33:56.616+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~13ff size 0x13ff (5119) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e84d0 blob([0x2c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e817e8540 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e85b0 blob([!~1000,0x2d000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,3ff]) SharedBlob(0x558e817e8620 sbid 0x0)) need 0x1000~3ff cache has 0x[1000~3ff] 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:39c0e826:::osdmap.18:0# 0x0~13ff = 5119 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 osd.0 18 add_map_bl 18 5119 bytes 2019-12-07T19:33:56.616+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 6 ==== PGlog(1.1s1 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x558e811b8800 con 0x558e818d7000 2019-12-07T19:33:56.616+0000 7fe2e6d3a700 15 osd.0 18 enqueue_peering_evt 1.1s1 epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info 2019-12-07T19:33:56.616+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.616+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 map 19 > 18 2019-12-07T19:33:56.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.616+0000 7fe2c425f700 20 osd.0 op_wq _add_slot_waiter 1.fs1 peering, item epoch is 19, will wait on OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 map 19 > 18 2019-12-07T19:33:56.616+0000 7fe2c7265700 20 osd.0 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 19, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 osd.0 18 got_full_map 19, nothing requested 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 osd.0 18 handle_osd_map pg_num_history pg_num_history(e19 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps purged_snaps {19={}} 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 osd.0 18 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e19 [1,19] lci=[13,19]) 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e8106c000 seq 12 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:ca5b972f:::inc_osdmap.19:0# key 0x7f7fffffffffffffffca5b97'/!inc_osdmap.19!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:56.616+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:ca5b972f:::inc_osdmap.19:0# 0x0~285 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1066 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:ca5b972f:::inc_osdmap.19:0# 0x0~285 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~285 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xd7b on front/back, now 0x0~1000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 1 blobs 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x2e000~1000/1000,1000,0 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x2e000~1000] 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81867570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e818664d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81867570 blob([0x2e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e818664d0 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.blob(0x558e81867570) get_ref 0x0~285 Blob(0x558e81867570 blob([0x2e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e818664d0 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.blob(0x558e81867570) get_ref init 0x1000, 1000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~285: 0x0~285 Blob(0x558e81867570 blob([0x2e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x285) SharedBlob(0x558e818664d0 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e818664e8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x285 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b6d88) dirty_range mark inline shard dirty 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:ca5b972f:::inc_osdmap.19:0# 0x0~285 = 0 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:34c0e826:::osdmap.19:0# key 0x7f7fffffffffffffff34c0e8'&!osdmap.19!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:56.616+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:34c0e826:::osdmap.19:0# 0x0~140c 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1067 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:34c0e826:::osdmap.19:0# 0x0~140c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~40c 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbf4 on front/back, now 0x0~1000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 2 blobs 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x2f000~2000/1000,2000,0 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x2f000~2000] 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e818669a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d1110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e818669a0 blob([0x2f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d1110 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.blob(0x558e818669a0) get_ref 0x0~1000 Blob(0x558e818669a0 blob([0x2f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d1110 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.blob(0x558e818669a0) get_ref init 0x1000, 1000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e818669a0 blob([0x2f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e811d1110 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e811d1128 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81867110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81867c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81867110 blob([!~1000,0x30000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81867c00 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.blob(0x558e81867110) get_ref 0x1000~40c Blob(0x558e81867110 blob([!~1000,0x30000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e81867c00 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.blob(0x558e81867110) get_ref init 0x2000, 1000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~40c: 0x1000~40c Blob(0x558e81867110 blob([!~1000,0x30000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x558e81867c00 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81867c18 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x140c 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7008) dirty_range mark inline shard dirty 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:34c0e826:::osdmap.19:0# 0x0~140c = 0 2019-12-07T19:33:56.616+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 0 blobs 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:33:56.616+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:56.616+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:56.616+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 0 blobs 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c000 cost 23258 (4 ios * 4000 + 7258 bytes) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c000 onodes 0x558e80251b80,0x558e817b9180,0x558e819b6c80,0x558e819b6f00 shared_blobs 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b6d88) update #-1:ca5b972f:::inc_osdmap.19:0# 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b6d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:ca5b972f:::inc_osdmap.19:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7008) update #-1:34c0e826:::osdmap.19:0# 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7008) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:34c0e826:::osdmap.19:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c000 allocated 0x[2e000~3000] released 0x[] 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 freelist allocate 0x2e000~3000 2019-12-07T19:33:56.616+0000 7fe2d959f700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 prepare 2019-12-07T19:33:56.616+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8106c000 2019-12-07T19:33:56.616+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 3 ==== PGlog(1.4s1 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x558e811b9000 con 0x558e818fe000 2019-12-07T19:33:56.616+0000 7fe2e753b700 15 osd.0 18 enqueue_peering_evt 1.4s1 epoch_sent: 19 epoch_requested: 18 MLogRec from 7(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) +create_info 2019-12-07T19:33:56.616+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 7(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 7(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.616+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.616+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 7(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.616+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 map 19 > 18 2019-12-07T19:33:56.616+0000 7fe2c5a62700 20 osd.0 op_wq _add_slot_waiter 1.4s1 peering, item epoch is 19, will wait on OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 7(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.616+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.617+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 5 ==== PGlog(1.ds5 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x558e811b9800 con 0x558e818d6000 2019-12-07T19:33:56.617+0000 7fe2e753b700 15 osd.0 18 enqueue_peering_evt 1.ds5 epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info 2019-12-07T19:33:56.617+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.617+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.617+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.617+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 map 19 > 18 2019-12-07T19:33:56.617+0000 7fe2c9269700 20 osd.0 op_wq _add_slot_waiter 1.ds5 peering, item epoch is 19, will wait on OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.617+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.617+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 aio_wait 2019-12-07T19:33:56.617+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c000 2019-12-07T19:33:56.617+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 io_done 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.617+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 9 ==== PGlog(1.9s3 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x558e819d8000 con 0x558e818d7c00 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:56.617+0000 7fe2e6d3a700 15 osd.0 18 enqueue_peering_evt 1.9s3 epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info 2019-12-07T19:33:56.617+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.617+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 10 ==== PGlog(1.es4 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x558e819d8800 con 0x558e818d7c00 2019-12-07T19:33:56.617+0000 7fe2e6d3a700 15 osd.0 18 enqueue_peering_evt 1.es4 epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info 2019-12-07T19:33:56.617+0000 7fe2e6d3a700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6c80 had 1 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6f00 had 1 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:33:56.617+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3860725760 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:56.617+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998762) bluestore 86 GiB free (0.959998), bluefs_ratio 0.039953 2019-12-07T19:33:56.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:33:56.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:33:56.617+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11234~226c to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.617+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11234 0x11234~226c to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x1234 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x234 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x4a0 and padding block with 0xb60 2019-12-07T19:33:56.617+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.617+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.617+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.617+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 map 19 > 18 2019-12-07T19:33:56.617+0000 7fe2c4a60700 20 osd.0 op_wq _add_slot_waiter 1.es4 peering, item epoch is 19, will wait on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.617+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x134a0 2019-12-07T19:33:56.617+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000288246s (0.000059049s flush + 0.000229197s kv commit) 2019-12-07T19:33:56.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:56.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8106c000> 2019-12-07T19:33:56.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 kv_submitted 2019-12-07T19:33:56.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c000 2019-12-07T19:33:56.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e8106c000 osr 0x558e8107a500 2019-12-07T19:33:56.617+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de3c0) _discard drop 9 0x1b000~1000 2019-12-07T19:33:56.617+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de3c0) prepare_write seq 10 0x1b000~1000 crc 5f344e98 2019-12-07T19:33:56.617+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de3c0) _discard drop 9 0x2000~1000 2019-12-07T19:33:56.617+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e811de3c0) prepare_write seq 10 0x2000~1000 crc 1cd2c56e 2019-12-07T19:33:56.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.617+0000 7fe2d7285700 10 osd.0 18 _committed_osd_maps 19..19 2019-12-07T19:33:56.617+0000 7fe2d7285700 10 osd.0 18 advance to epoch 19 (<= last 19 <= newest_map 19) 2019-12-07T19:33:56.617+0000 7fe2d7285700 7 osd.0 19 consume_map version 19 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:1.identify_splits_and_merges identify_splits_and_merges slot 1.9s3 has no pg and waiting_for_split 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:1.identify_splits_and_merges identify_splits_and_merges slot 1.1s1 has no pg and waiting_for_split 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:2.identify_splits_and_merges identify_splits_and_merges slot 1.as3 has no pg and waiting_for_split 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0 19 identify_splits_and_merges 1.2s0 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:3.identify_splits_and_merges identify_splits_and_merges slot 1.bs3 has no pg and waiting_for_split 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:4.identify_splits_and_merges identify_splits_and_merges slot 1.4s1 has no pg and waiting_for_split 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:5.identify_splits_and_merges identify_splits_and_merges slot 1.ds5 has no pg and waiting_for_split 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:6.identify_splits_and_merges identify_splits_and_merges slot 1.es4 has no pg and waiting_for_split 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:7.identify_splits_and_merges identify_splits_and_merges slot 1.7s3 has no pg and waiting_for_split 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:7.identify_splits_and_merges identify_splits_and_merges slot 1.fs1 has no pg and waiting_for_split 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0 19 prune_pg_created 2019-12-07T19:33:56.617+0000 7fe2d7285700 10 osd.0:0.consume_map 19 (was 18) 2019-12-07T19:33:56.617+0000 7fe2d7285700 10 osd.0:1.consume_map 19 (was 18) 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 empty, pruning 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 pending_peering first epoch 19 <= 19, requeueing 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {19=} 2019-12-07T19:33:56.617+0000 7fe2d7285700 10 osd.0:2.consume_map 19 (was 18) 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 empty, pruning 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 pending_peering first epoch 19 <= 19, requeueing 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.2s0 to_process <> waiting <> waiting_peering {19=} 2019-12-07T19:33:56.617+0000 7fe2d7285700 10 osd.0:3.consume_map 19 (was 18) 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 pending_peering first epoch 19 <= 19, requeueing 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.bs3 to_process <> waiting <> waiting_peering {19=} 2019-12-07T19:33:56.617+0000 7fe2d7285700 10 osd.0:4.consume_map 19 (was 18) 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 pending_peering first epoch 19 <= 19, requeueing 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:4._wake_pg_slot _wake_pg_slot 1.4s1 to_process <> waiting <> waiting_peering {19=} 2019-12-07T19:33:56.617+0000 7fe2d7285700 10 osd.0:5.consume_map 19 (was 18) 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 pending_peering first epoch 19 <= 19, requeueing 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:5._wake_pg_slot _wake_pg_slot 1.ds5 to_process <> waiting <> waiting_peering {19=} 2019-12-07T19:33:56.617+0000 7fe2d7285700 10 osd.0:6.consume_map 19 (was 18) 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 pending_peering first epoch 19 <= 19, requeueing 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {19=} 2019-12-07T19:33:56.617+0000 7fe2d7285700 10 osd.0:7.consume_map 19 (was 18) 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 empty, pruning 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 pending_peering first epoch 19 <= 19, requeueing 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.fs1 to_process <> waiting <> waiting_peering {19=} 2019-12-07T19:33:56.617+0000 7fe2d7285700 10 osd.0 19 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:56.617+0000 7fe2d7285700 15 osd.0 19 enqueue_peering_evt 1.2s0 epoch_sent: 19 epoch_requested: 19 NullEvt 2019-12-07T19:33:56.617+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) 2019-12-07T19:33:56.617+0000 7fe2d7285700 7 osd.0 19 activate_map version 19 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt +create_info) prio 255 cost 10 e19) pg 0x558e81075000 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.2s0 18 -> 19 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] handle_advance_map [0,4,1,6,5,3]/[0,4,1,6,5,3] -- 0/0 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Peering advmap 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] adjust_need_up_thru now 18, need_up_thru now false 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Started advmap 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.618+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] hit_set_clear 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] agent_stop 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] activate_map 2019-12-07T19:33:56.618+0000 7fe2caa6c700 7 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: handle ActMap primary 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.618+0000 7fe2caa6c700 15 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] publish_stats_to_osd 0:0 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.618+0000 7fe2caa6c700 5 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering/WaitUpThru 2019-12-07T19:33:56.618852+0000 2 0.000278 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 no pg, should create on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] state: Leaving Peering 2019-12-07T19:33:56.618+0000 7fe2caa6c700 5 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+peering mbc={}] exit Started/Primary/Peering 2019-12-07T19:33:56.618869+0000 0 0.000000 2019-12-07T19:33:56.618+0000 7fe2caa6c700 5 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] enter Started/Primary/Active 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] state: In Active, about to call activate 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 19 _make_pg 1.1s1 2019-12-07T19:33:56.618+0000 7fe2cc26f700 20 osd.0 19 agent_entry empty queue 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:33:56.618+0000 7fe2cb26d700 5 osd.0 pg_epoch: 18 pg[1.1(unlocked)] enter Initial 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.6 ruub 0s 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.073596236s) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.1(2) 1.2s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.1(2) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.1(2) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 pg_epoch: 18 pg[1.1(unlocked)] enter NotTrimming 2019-12-07T19:33:56.618+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- PGlog(1.2s2 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.073596236s int 16s) e19/18) v6 -- 0x558e819d9800 con 0x558e818d7c00 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach 1.1s1_head fresh osr 0x558e8107be00 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 18 pg[1.1s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 1 up [3,0,6,4,5,1] acting [3,0,6,4,5,1] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.618+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 11 ==== PGlog(1.as3 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075332677s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x558e819d9000 con 0x558e818d7c00 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.1(2) 1.2s2( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.618+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.as3 epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075332677s int 16s) +create_info 2019-12-07T19:33:56.618+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075332677s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558e811d1570] 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.3(5) 1.2s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.073708008s + 0s) 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.3(5) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.3(5) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.618+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- PGlog(1.2s5 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.073596236s int 16s) e19/18) v6 -- 0x558e819da000 con 0x558e818d7000 2019-12-07T19:33:56.618+0000 7fe2cb26d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.618+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.3(5) 1.2s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.4(1) 1.2s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 5 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.619063+0000 0 0.000000 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.4(1) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.618+0000 7fe2cb26d700 5 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.4(1) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.618+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- PGlog(1.2s1 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.073596236s int 16s) e19/18) v6 -- 0x558e819da800 con 0x558e818fe400 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.4(1) 1.2s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.5(4) 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 5 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.619104+0000 1 0.000031 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.5(4) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.618+0000 7fe2cb26d700 5 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 no pg, should create on OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.5(4) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 5 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.618+0000 7fe2cb26d700 1 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.618+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- PGlog(1.2s4 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.073596236s int 16s) e19/18) v6 -- 0x558e819db000 con 0x558e818d6800 2019-12-07T19:33:56.618+0000 7fe2cb26d700 5 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.619125+0000 0 0.000000 2019-12-07T19:33:56.618+0000 7fe2cb26d700 5 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.5(4) 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.6(3) 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 osd.0 19 _make_pg 1.ds5 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e819a1b00 seq 1 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.6(3) is up to date, but sending pg_log anyway 2019-12-07T19:33:56.618+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.1s1_head bits 4 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.6(3) sending log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.1s1_head bits 4 = 0 2019-12-07T19:33:56.618+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- PGlog(1.2s3 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.073596236s int 16s) e19/18) v6 -- 0x558e819db800 con 0x558e818ffc00 2019-12-07T19:33:56.618+0000 7fe2c5261700 5 osd.0 pg_epoch: 18 pg[1.d(unlocked)] enter Initial 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) get_onode oid 1#1:80000000::::head# key 0x8180000000000000018000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) r -2 v.len 0 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate peer osd.6(3) 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) uptodate 2019-12-07T19:33:56.618+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 0(0) 2019-12-07T19:33:56.618+0000 7fe2c5261700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1068 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:33:56.618+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 1(2) 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 3(5) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._infover' <- _infover 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 4(1) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:33:56.618+0000 7fe2c5261700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.618+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 5(4) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] activate setting up missing_loc from shard 6(3) 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating mbc={}] needs_recovery is recovered 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.618+0000 7fe2c5261700 20 osd.0 pg_epoch: 18 pg[1.d(unlocked)] enter NotTrimming 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.618+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach 1.ds5_head fresh osr 0x558e817952c0 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1b00 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1b00 onodes 0x558e81a02000 shared_blobs 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e81a02108) update 1#1:80000000::::head# 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e81a02108) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:80000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 osd.0 pg_epoch: 18 pg[1.ds5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 5 up [2,1,7,3,4,0] acting [2,1,7,3,4,0] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1b00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(4),6(3) upset 0(0),1(2),3(5),4(1),5(4),6(3) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 0(0) primary objects 0 missing 0 2019-12-07T19:33:56.618+0000 7fe2c5261700 20 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 prepare 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1b00 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 io_done 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.618+0000 7fe2c5261700 20 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558e811d1b90] 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(1) objects 0 missing 0 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 19 handle_pg_create_info new pg pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.073970922s + 0s) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0:1._attach_pg 1.1s1 0x558e819e4000 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(4) objects 0 missing 0 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 19 identify_splits_and_merges 1.1s1 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0:1._prime_splits 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 6(3) objects 0 missing 0 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.1s1 18 -> 19 2019-12-07T19:33:56.618+0000 7fe2c5261700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 18 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [3,0,6,4,5,1]/[3,0,6,4,5,1] -- 3/3 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.618+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.618+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.618+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.618+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.618+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074132637s int 16s) +create_info 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.3(0) 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 merge_log log((0'0,0'0], crt=0'0) from osd.3(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.074132637s int 16s) 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.002764194s -> ruub 36.071368443s 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.071368443s 2019-12-07T19:33:56.618+0000 7fe2cb26d700 5 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.619569+0000 3 0.000061 2019-12-07T19:33:56.618+0000 7fe2c5261700 5 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.619327+0000 0 0.000000 2019-12-07T19:33:56.618+0000 7fe2cb26d700 5 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.618+0000 7fe2cb26d700 5 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.618+0000 7fe2c5261700 5 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.618+0000 7fe2c5261700 20 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.618+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e81a0a300 seq 2 2019-12-07T19:33:56.618+0000 7fe2c5261700 10 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.618+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:33:56.618+0000 7fe2c5261700 5 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.619644+0000 1 0.000303 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:33:56.618+0000 7fe2c5261700 5 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.618+0000 7fe2c5261700 5 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.618+0000 7fe2c5261700 1 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:56.618+0000 7fe2c5261700 5 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.619679+0000 0 0.000000 2019-12-07T19:33:56.619+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.619+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.619+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:33:56.619+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:56.619+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:56.619+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 no pg, should create on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.619+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x134a0~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.619+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x134a0 0x134a0~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.618+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.619+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.619+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x34a0 2019-12-07T19:33:56.619+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x4a0 2019-12-07T19:33:56.619+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.619+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xa0d and padding block with 0x5f3 2019-12-07T19:33:56.618+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:56.618+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 7(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 7(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.619+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a300 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.619+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a300 onodes shared_blobs 2019-12-07T19:33:56.619+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 osd.0 19 _make_pg 1.es4 2019-12-07T19:33:56.618+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 prepare 2019-12-07T19:33:56.618+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a300 2019-12-07T19:33:56.619+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 io_done 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7fe2c8a68700 5 osd.0 pg_epoch: 18 pg[1.e(unlocked)] enter Initial 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.619+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 no pg, should create on OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 no pg, should create on OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7fe2c8267700 10 osd.0 19 _make_pg 1.fs1 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 19 _make_pg 1.bs3 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.619+0000 7fe2ca26b700 5 osd.0 pg_epoch: 18 pg[1.b(unlocked)] enter Initial 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 no pg, should create on OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.f(unlocked)] enter Initial 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 no pg, should create on OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 7(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 19 _make_pg 1.9s3 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 10 osd.0 19 _make_pg 1.4s1 2019-12-07T19:33:56.619+0000 7fe2c8267700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.619+0000 7fe2c7265700 5 osd.0 pg_epoch: 18 pg[1.9(unlocked)] enter Initial 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 5 osd.0 pg_epoch: 18 pg[1.4(unlocked)] enter Initial 2019-12-07T19:33:56.619+0000 7fe2c8267700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.619+0000 7fe2c8267700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.619+0000 7fe2c8267700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.619+0000 7fe2c8267700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0 pg_epoch: 18 pg[1.9(unlocked)] enter NotTrimming 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach 1.9s3_head fresh osr 0x558e811de640 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 pg_epoch: 18 pg[1.b(unlocked)] enter NotTrimming 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach 1.bs3_head fresh osr 0x558e81795cc0 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 18 pg[1.9s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 3 up [1,4,5,0,7,6] acting [1,4,5,0,7,6] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 18 pg[1.bs3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 3 up [4,6,2,0,1,5] acting [4,6,2,0,1,5] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.074677946s + 0s) 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558e811d0af0] 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.074685999s + 0s) 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.619+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.619+0000 7fe2ca26b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.619+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs3_head options 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.619+0000 7fe2c7265700 5 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.620030+0000 0 0.000000 2019-12-07T19:33:56.619+0000 7fe2c7265700 5 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.619+0000 7fe2ca26b700 5 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.620038+0000 0 0.000000 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.619+0000 7fe2ca26b700 5 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.619+0000 7fe2c7265700 5 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.620074+0000 1 0.000035 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.619+0000 7fe2c7265700 5 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.619+0000 7fe2ca26b700 5 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.620084+0000 1 0.000036 2019-12-07T19:33:56.619+0000 7fe2c7265700 5 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.619+0000 7fe2ca26b700 5 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.619+0000 7fe2c7265700 1 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.619+0000 7fe2ca26b700 5 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.619+0000 7fe2c7265700 5 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.620097+0000 0 0.000000 2019-12-07T19:33:56.619+0000 7fe2ca26b700 1 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.619+0000 7fe2c7265700 5 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.619+0000 7fe2ca26b700 5 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.620106+0000 0 0.000000 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7fe2ca26b700 5 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e81a0b500 seq 1 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.9s3_head bits 4 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e81a0bb00 seq 1 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.9s3_head bits 4 = 0 2019-12-07T19:33:56.619+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.bs3_head bits 4 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.bs3_head bits 4 = 0 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) get_onode oid 3#1:90000000::::head# key 0x8380000000000000019000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs3_head 0x558e819c41e0) get_onode oid 3#1:d0000000::::head# key 0x838000000000000001d000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) r -2 v.len 0 2019-12-07T19:33:56.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1069 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs3_head 0x558e819c41e0) r -2 v.len 0 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:56.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:56.619+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1070 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._infover' <- _infover 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:56.619+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:56.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:56.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._infover' <- _infover 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:56.619+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:56.619+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b500 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b500 onodes 0x558e81a02280 shared_blobs 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e81a02388) update 3#1:90000000::::head# 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e81a02388) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0bb00 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:90000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0bb00 onodes 0x558e81a02500 shared_blobs 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e81a02608) update 3#1:d0000000::::head# 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b500 allocated 0x[] released 0x[] 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e81a02608) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:d0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0bb00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 prepare 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b500 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 io_done 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 prepare 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0bb00 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 io_done 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 19 handle_pg_create_info new pg pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0:1._attach_pg 1.9s3 0x558e8192b000 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7fe2c8a68700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0 19 identify_splits_and_merges 1.9s3 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0:1._prime_splits 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 19 handle_pg_create_info new pg pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.619+0000 7fe2c8a68700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0:3._attach_pg 1.bs3 0x558e81928000 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 19 identify_splits_and_merges 1.bs3 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0:3._prime_splits 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.9s3 18 -> 19 2019-12-07T19:33:56.619+0000 7fe2c8a68700 20 osd.0 pg_epoch: 18 pg[1.e(unlocked)] enter NotTrimming 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 18 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach 1.es4_head fresh osr 0x558e811df7c0 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs3 18 -> 19 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 osd.0 pg_epoch: 18 pg[1.es4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 4 up [1,3,4,5,0,6] acting [1,3,4,5,0,6] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 18 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [4,6,2,0,1,5]/[4,6,2,0,1,5] -- 4/4 2019-12-07T19:33:56.619+0000 7fe2c8a68700 20 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.619+0000 7fe2c8a68700 20 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.619+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs3_head options 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.074994052s + 0s) 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.619+0000 7fe2c7265700 20 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074537935s int 16s) +create_info 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.619+0000 7fe2c8a68700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 merge_log log((0'0,0'0], crt=0'0) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.619+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 4(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.063561597s int 16s) +create_info 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.4(0) 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 merge_log log((0'0,0'0], crt=0'0) from osd.4(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.063561597s int 16s) 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 18446744073.703827048s -> ruub 36.069286165s 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.069286165s 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.619+0000 7fe2ca26b700 5 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.620396+0000 3 0.000049 2019-12-07T19:33:56.619+0000 7fe2ca26b700 5 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.619+0000 7fe2ca26b700 5 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.619+0000 7fe2c8a68700 5 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.620405+0000 0 0.000000 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.619+0000 7fe2c8a68700 5 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.619+0000 7fe2c8a68700 20 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e8195e300 seq 2 2019-12-07T19:33:56.619+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e300 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e300 onodes shared_blobs 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 prepare 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e300 2019-12-07T19:33:56.619+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 io_done 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.619+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 12 ==== PGlog(1.7s3 log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) e19/18) v6 ==== 1123+0+0 (unknown 0 0 0) 0x558e81960000 con 0x558e818d7c00 2019-12-07T19:33:56.619+0000 7fe2c8a68700 5 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.620506+0000 1 0.000045 2019-12-07T19:33:56.619+0000 7fe2c8a68700 5 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.619+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.7s3 epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info 2019-12-07T19:33:56.619+0000 7fe2c8a68700 5 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 18 pg[1.4(unlocked)] enter NotTrimming 2019-12-07T19:33:56.619+0000 7fe2c8a68700 1 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach 1.4s1_head fresh osr 0x558e811df900 2019-12-07T19:33:56.619+0000 7fe2c8a68700 5 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.620540+0000 0 0.000000 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 18 pg[1.4s1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 1 up [7,0,1,5,2,4] acting [7,0,1,5,2,4] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.619+0000 7fe2c8a68700 5 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.619+0000 7fe2c8a68700 20 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558e811d1b20] 2019-12-07T19:33:56.619+0000 7fe2c5261700 5 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.075271148s + 0s) 2019-12-07T19:33:56.619+0000 7fe2c5261700 20 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.619+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:33:56.619+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x13a0d 2019-12-07T19:33:56.619+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:56.619+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.619+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e8195ef00 seq 1 2019-12-07T19:33:56.619+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.619+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.619+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.619+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e8195f500 seq 1 2019-12-07T19:33:56.619+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.ds5_head bits 4 2019-12-07T19:33:56.619+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.ds5_head bits 4 = 0 2019-12-07T19:33:56.619+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.619+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.001070512s (0.000000099s flush + 0.001070413s kv commit) 2019-12-07T19:33:56.619+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:33:56.619+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.ds5_head 0x558e817b5860) get_onode oid 5#1:b0000000::::head# key 0x858000000000000001b000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.ds5_head 0x558e817b5860) r -2 v.len 0 2019-12-07T19:33:56.620+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1071 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:56.620+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._infover' <- _infover 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:56.620+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:56.620+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195f500 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195f500 onodes 0x558e81a02780 shared_blobs 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore.extentmap(0x558e81a02888) update 5#1:b0000000::::head# 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore.extentmap(0x558e81a02888) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:b0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195f500 allocated 0x[] released 0x[] 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 prepare 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195f500 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 io_done 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0 19 handle_pg_create_info new pg pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0:5._attach_pg 1.ds5 0x558e819e7000 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 osd.0:5._wake_pg_slot _wake_pg_slot 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 osd.0 19 identify_splits_and_merges 1.ds5 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0:5._prime_splits 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 osd.0:5.update_pg_epoch 1.ds5 18 -> 19 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0 pg_epoch: 18 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.620+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 osd.0 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 2(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.088857038s int 16s) +create_info 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.2(0) 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 merge_log log((0'0,0'0], crt=0'0) from osd.2(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.088857038s int 16s) 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.016960406s -> ruub 36.071896632s 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.071896632s 2019-12-07T19:33:56.620+0000 7fe2c5261700 5 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.621051+0000 3 0.000948 2019-12-07T19:33:56.620+0000 7fe2c5261700 5 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.620+0000 7fe2c5261700 5 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e8195f800 seq 2 2019-12-07T19:33:56.620+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195f800 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195f800 onodes shared_blobs 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195f800 allocated 0x[] released 0x[] 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 prepare 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195f800 2019-12-07T19:33:56.620+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 io_done 2019-12-07T19:33:56.620+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.074537935s int 16s) 2019-12-07T19:33:56.620+0000 7fe2c7265700 20 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.002166438s -> ruub 36.072371497s 2019-12-07T19:33:56.620+0000 7fe2c7265700 20 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.072371497s 2019-12-07T19:33:56.620+0000 7fe2c7265700 5 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.621314+0000 3 0.000046 2019-12-07T19:33:56.620+0000 7fe2c7265700 5 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.620+0000 7fe2c7265700 5 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.620+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.620+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.620+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.620+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:33:56.620+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:33:56.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e81a0b800 seq 2 2019-12-07T19:33:56.620+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:56.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:33:56.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:33:56.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:56.620+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:56.620+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b800 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b800 onodes shared_blobs 2019-12-07T19:33:56.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b800 allocated 0x[] released 0x[] 2019-12-07T19:33:56.620+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 prepare 2019-12-07T19:33:56.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b800 2019-12-07T19:33:56.620+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 io_done 2019-12-07T19:33:56.619+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.es4_head bits 4 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.es4_head bits 4 = 0 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.es4_head 0x558e819c4960) get_onode oid 4#1:70000000::::head# key 0x8480000000000000017000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.es4_head 0x558e819c4960) r -2 v.len 0 2019-12-07T19:33:56.620+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1072 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:56.620+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._infover' <- _infover 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:56.620+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:56.620+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195ef00 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195ef00 onodes 0x558e81a02a00 shared_blobs 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore.extentmap(0x558e81a02b08) update 4#1:70000000::::head# 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore.extentmap(0x558e81a02b08) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 4#1:70000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195ef00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 prepare 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195ef00 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 io_done 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 osd.0 19 handle_pg_create_info new pg pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 osd.0:6._attach_pg 1.es4 0x558e81a98000 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 osd.0:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 osd.0 19 identify_splits_and_merges 1.es4 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 osd.0:6._prime_splits 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 osd.0:6.update_pg_epoch 1.es4 18 -> 19 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 osd.0 pg_epoch: 18 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [1,3,4,5,0,6]/[1,3,4,5,0,6] -- 1/1 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.620+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:33:56.620+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 osd.0 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074554249s int 16s) +create_info 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 merge_log log((0'0,0'0], crt=0'0) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.074554249s int 16s) 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.002166438s -> ruub 36.072387811s 2019-12-07T19:33:56.620+0000 7fe2c8a68700 20 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.072387811s 2019-12-07T19:33:56.620+0000 7fe2c8a68700 5 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.621663+0000 3 0.000066 2019-12-07T19:33:56.620+0000 7fe2c8a68700 5 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.620+0000 7fe2c8a68700 5 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.620+0000 7fe2c8a68700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.621+0000 7fe2c8a68700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.621+0000 7fe2c8a68700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.621+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:33:56.621+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:33:56.621+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e8195f200 seq 2 2019-12-07T19:33:56.621+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:56.621+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:33:56.621+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:33:56.621+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:56.621+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:56.621+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195f200 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.621+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195f200 onodes shared_blobs 2019-12-07T19:33:56.621+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195f200 allocated 0x[] released 0x[] 2019-12-07T19:33:56.621+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f200 prepare 2019-12-07T19:33:56.621+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195f200 2019-12-07T19:33:56.621+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f200 io_done 2019-12-07T19:33:56.621+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1b00> 2019-12-07T19:33:56.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.621+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 kv_submitted 2019-12-07T19:33:56.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1b00 2019-12-07T19:33:56.621+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 finishing 2019-12-07T19:33:56.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1b00 onodes 0x558e81a02000 2019-12-07T19:33:56.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1b00 done 2019-12-07T19:33:56.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a300 kv_queued 2019-12-07T19:33:56.621+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1b00 [] 2019-12-07T19:33:56.621+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.619+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 5 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.621842+0000 0 0.000000 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 5 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 5 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.621884+0000 1 0.000034 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 5 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 5 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 1 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.619+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.618+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(4) missing= 0 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 6(3) missing= 0 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.621+0000 7fe2caa6c700 15 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 18:1 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: Activate Finished 2019-12-07T19:33:56.621+0000 7fe2caa6c700 5 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] enter Started/Primary/Active/Activating 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 NullEvt +create_info 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e8195fb00 seq 3 2019-12-07T19:33:56.621+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195fb00 cost 5345 (1 ios * 4000 + 1345 bytes) 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195fb00 onodes shared_blobs 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195fb00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 prepare 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195fb00 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 io_done 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 NullEvt) prio 255 cost 10 e19) pg 0x558e81075000 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 NullEvt 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075332677s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075332677s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 no pg, should create on OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075332677s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 osd.0 19 _make_pg 1.as3 2019-12-07T19:33:56.621+0000 7fe2caa6c700 5 osd.0 pg_epoch: 18 pg[1.a(unlocked)] enter Initial 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.621+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.619+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:33:56.621+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 pg_epoch: 18 pg[1.a(unlocked)] enter NotTrimming 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach 1.as3_head fresh osr 0x558e81966000 2019-12-07T19:33:56.621+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2468 < min_flush_size 524288 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.as3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 3 up [1,6,2,0,5,4] acting [1,6,2,0,5,4] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.621+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 2 2019-12-07T19:33:56.621+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3834 < min_flush_size 524288 2019-12-07T19:33:56.621+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 2 2019-12-07T19:33:56.621+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4936 < min_flush_size 524288 2019-12-07T19:33:56.621+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:56.621+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4959 < min_flush_size 524288 2019-12-07T19:33:56.621+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4959 < min_flush_size 524288 2019-12-07T19:33:56.621+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.621+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x13a0d~135f to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.621+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x13a0d 0x13a0d~135f to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.621+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.621+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x3a0d 2019-12-07T19:33:56.621+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xa0d 2019-12-07T19:33:56.621+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.621+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd6c and padding block with 0x294 2019-12-07T19:33:56.621+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 5 ==== pg_info2(1.2s0 1.2s0 1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (unknown 0 0 0) 0x558e80ff2a00 con 0x558e818ffc00 2019-12-07T19:33:56.621+0000 7fe2e7d3c700 15 osd.0 19 enqueue_peering_evt 1.2s0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 6(3) info: 1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s) 2019-12-07T19:33:56.621+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 6(3) info: 1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.621+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 4 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (unknown 0 0 0) 0x558e81909b00 con 0x558e818ffc00 2019-12-07T19:33:56.621+0000 7fe2d4a80700 20 osd.0 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.621+0000 7fe2d4a80700 10 osd.0 19 do_waiters -- start 2019-12-07T19:33:56.621+0000 7fe2d4a80700 10 osd.0 19 do_waiters -- finish 2019-12-07T19:33:56.621+0000 7fe2d4a80700 20 osd.0 19 _dispatch 0x558e81909b00 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.621+0000 7fe2caa6c700 20 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.077319850s + 0s) 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.621+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.621+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.621+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x14d6c 2019-12-07T19:33:56.621+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.621+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.621+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.621+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.621+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 4 cleaned 0 in 0.002056861s (0.000000092s flush + 0.002056769s kv commit) 2019-12-07T19:33:56.621+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 6 submitting 6 deferred done 0 stable 0 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 2 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2468 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3570 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4936 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 2 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 6038 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0a300,0x558e81a0b500,0x558e81a0bb00,0x558e8195e300> 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a300 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 7226 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 finishing 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a300 onodes 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a300 done 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a300 [] 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b500 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 finishing 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b500 onodes 0x558e81a02280 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b500 done 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b500 [] 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 7249 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0bb00 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 7249 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 finishing 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0bb00 onodes 0x558e81a02500 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0bb00 done 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0bb00 [] 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e300 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 finishing 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e300 onodes 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 done 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e300 [] 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.622+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.621+0000 7fe2c9a6a700 5 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.621906+0000 0 0.000000 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 5 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e819a1200 seq 1 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.4s1_head bits 4 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.4s1_head bits 4 = 0 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) get_onode oid 1#1:20000000::::head# key 0x8180000000000000012000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x14d6c~1c51 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) r -2 v.len 0 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x14d6c 0x14d6c~1c51 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.621+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x4d6c 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd6c 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x9bd and padding block with 0x643 2019-12-07T19:33:56.622+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.622+0000 7fe2caa6c700 5 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.622904+0000 0 0.000000 2019-12-07T19:33:56.622+0000 7fe2caa6c700 5 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.622+0000 7fe2caa6c700 20 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.622+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.622+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.622+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.622+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x169bd 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1073 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.622+0000 7fe2caa6c700 5 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.622947+0000 1 0.000035 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.622+0000 7fe2caa6c700 5 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.622+0000 7fe2caa6c700 5 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195f500,0x558e8195f800,0x558e81a0b800,0x558e8195ef00,0x558e8195f200,0x558e8195fb00> 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2caa6c700 1 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195f500 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 finishing 2019-12-07T19:33:56.622+0000 7fe2caa6c700 5 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.622986+0000 0 0.000000 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195f500 onodes 0x558e81a02780 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f500 done 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f800 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195f500 [] 2019-12-07T19:33:56.622+0000 7fe2caa6c700 5 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2caa6c700 20 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195f800 2019-12-07T19:33:56.622+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 6 cleaned 0 in 0.000280274s (0.000000094s flush + 0.000280180s kv commit) 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:56.622+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 finishing 2019-12-07T19:33:56.622+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e811fa300 seq 1 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195f800 onodes 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f800 done 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195f800 [] 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:33:56.622+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._infover' <- _infover 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:33:56.622+0000 7fe2c9269700 15 osd.0 19 enqueue_peering_evt 1.ds5 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:33:56.622+0000 7fe2c9269700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.622+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.as3_head bits 4 2019-12-07T19:33:56.622+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.as3_head bits 4 = 0 2019-12-07T19:33:56.622+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:33:56.622+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:33:56.622+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558e819e7000 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.622+0000 7fe2c9269700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes 0x558e81a02c80 shared_blobs 2019-12-07T19:33:56.622+0000 7fe2c9269700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e81a02d88) update 1#1:20000000::::head# 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e81a02d88) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:20000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.622+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[] released 0x[] 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b800 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 finishing 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b800 onodes 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 done 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b800 [] 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195ef00 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 finishing 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195ef00 onodes 0x558e81a02a00 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195ef00 done 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f200 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195ef00 [] 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f200 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195f200 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f200 finishing 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195f200 onodes 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f200 done 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195f200 [] 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195fb00 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 finishing 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195fb00 onodes 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 done 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195fb00 [] 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.622+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7fe2c9269700 10 osd.0 19 send_incremental_map 18 -> 19 to 0x558e818d6000 v2:172.21.15.131:6804/37077 2019-12-07T19:33:56.622+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:ca5b972f:::inc_osdmap.19:0# 0x0~0 2019-12-07T19:33:56.622+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~285 size 0x285 (645) 2019-12-07T19:33:56.622+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:56.622+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81867570 blob([0x2e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x285) SharedBlob(0x558e818664d0 sbid 0x0)) need 0x0~285 cache has 0x[0~285] 2019-12-07T19:33:56.622+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:ca5b972f:::inc_osdmap.19:0# 0x0~285 = 645 2019-12-07T19:33:56.622+0000 7fe2c9269700 10 osd.0 19 add_map_inc_bl 19 645 bytes 2019-12-07T19:33:56.622+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(19..19 src has 1..19) v4 -- 0x558e81222000 con 0x558e818d6000 2019-12-07T19:33:56.622+0000 7fe2c9269700 10 osd.0 19 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 18 -> 19 (shared) 2019-12-07T19:33:56.622+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.ds0 1.ds0 1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558e80ff3100 con 0x558e818d6000 2019-12-07T19:33:56.622+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 osd.0 19 handle_pg_create_info new pg pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 osd.0:4._attach_pg 1.4s1 0x558e81938000 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 osd.0:4._wake_pg_slot _wake_pg_slot 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:33:56.622+0000 7fe2c8a68700 15 osd.0 19 enqueue_peering_evt 1.es4 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:33:56.622+0000 7fe2c8a68700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x169bd~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x169bd 0x169bd~56d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x69bd 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x9bd 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xf2a and padding block with 0xd6 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 osd.0 19 identify_splits_and_merges 1.4s1 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.622+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x16f2a 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.622+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.622+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558e81a98000 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000112392s (0.000000099s flush + 0.000112293s kv commit) 2019-12-07T19:33:56.622+0000 7fe2c8a68700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:56.622+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7fe2c8a68700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.622+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1200> 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:33:56.622+0000 7fe2c8a68700 10 osd.0 19 send_incremental_map 18 -> 19 to 0x558e818d7c00 v2:172.21.15.131:6823/37078 2019-12-07T19:33:56.622+0000 7fe2ca26b700 15 osd.0 19 enqueue_peering_evt 1.bs3 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.622+0000 7fe2ca26b700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.622+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(19..19 src has 1..19) v4 -- 0x558e81957200 con 0x558e818d7c00 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 0x558e81a02c80 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df900 q now empty 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:33:56.622+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.622+0000 7fe2c8a68700 10 osd.0 19 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 18 -> 19 (shared) 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 osd.0:4._prime_splits 2019-12-07T19:33:56.622+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.es0 1.es0 1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558e80ff3800 con 0x558e818d7c00 2019-12-07T19:33:56.622+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.622+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.as3_head 0x558e819c5e00) get_onode oid 3#1:50000000::::head# key 0x8380000000000000015000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 osd.0:4.update_pg_epoch 1.4s1 18 -> 19 2019-12-07T19:33:56.622+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.as3_head 0x558e819c5e00) r -2 v.len 0 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 18 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:33:56.622+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.as3_head 3#1:50000000::::head# 2019-12-07T19:33:56.622+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1074 2019-12-07T19:33:56.622+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:33:56.622+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:33:56.622+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._infover' <- _infover 2019-12-07T19:33:56.622+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:33:56.622+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.as3_head 3#1:50000000::::head# 2019-12-07T19:33:56.622+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:33:56.622+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:33:56.622+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:33:56.622+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:33:56.622+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.622+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.622+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558e81928000 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:33:56.622+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.622+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.622+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:33:56.622+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.622+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.622+0000 7fe2ca26b700 10 osd.0 19 send_incremental_map 18 -> 19 to 0x558e818fe400 v2:172.21.15.109:6810/36938 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 osd.0 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.622+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(19..19 src has 1..19) v4 -- 0x558e81956900 con 0x558e818fe400 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.622+0000 7fe2ca26b700 10 osd.0 19 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 18 -> 19 (shared) 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:33:56.622+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.bs0 1.bs0 1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558e81228000 con 0x558e818fe400 2019-12-07T19:33:56.622+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.622+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.622+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:33:56.622+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 7(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.052429711s int 16s) +create_info 2019-12-07T19:33:56.622+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.7(0) 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 merge_log log((0'0,0'0], crt=0'0) from osd.7(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.622+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.052429711s int 16s) 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 18446744073.690142012s -> ruub 36.071839315s 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.071839315s 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 5 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.623706+0000 3 0.001005 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 5 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 5 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e819a1500 seq 2 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1500 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1500 onodes shared_blobs 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1500 allocated 0x[] released 0x[] 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 prepare 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1500 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 io_done 2019-12-07T19:33:56.623+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x16f2a~465 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x16f2a 0x16f2a~465 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x6f2a 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xf2a 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x38f and padding block with 0xc71 2019-12-07T19:33:56.622+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa300 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa300 onodes 0x558e81a02f00 shared_blobs 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e81a03008) update 3#1:50000000::::head# 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e81a03008) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:50000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 prepare 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa300 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 io_done 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 19 handle_pg_create_info new pg pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0:2._attach_pg 1.as3 0x558e8193b000 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 19 identify_splits_and_merges 1.as3 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0:2._prime_splits 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.as3 18 -> 19 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 18 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [1,6,2,0,5,4]/[1,6,2,0,5,4] -- 1/1 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.623+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075332677s int 16s) +create_info 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 merge_log log((0'0,0'0], crt=0'0) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.075332677s int 16s) 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.002166438s -> ruub 36.073166239s 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.073166239s 2019-12-07T19:33:56.623+0000 7fe2caa6c700 5 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.624059+0000 3 0.000062 2019-12-07T19:33:56.623+0000 7fe2caa6c700 5 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.623+0000 7fe2caa6c700 5 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.619+0000 7fe2c8267700 20 osd.0 pg_epoch: 18 pg[1.f(unlocked)] enter NotTrimming 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach 1.fs1_head fresh osr 0x558e81966640 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e8195e300 seq 2 2019-12-07T19:33:56.623+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.fs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 1 up [3,0,5,4,1,6] acting [3,0,5,4,1,6] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e300 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e300 onodes shared_blobs 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 prepare 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558e811d1570] 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e300 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 io_done 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.078831741s + 0s) 2019-12-07T19:33:56.623+0000 7fe2caa6c700 15 osd.0 19 enqueue_peering_evt 1.2s0 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558e81075000 2019-12-07T19:33:56.623+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: _activate_committed 19 peer_activated now 0(0) last_interval_started 0 last_epoch_started 0 same_interval_since 18 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.623+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.624186+0000 0 0.000000 2019-12-07T19:33:56.623+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1738f 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.623+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.624226+0000 1 0.000031 2019-12-07T19:33:56.623+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000420610s (0.000000103s flush + 0.000420507s kv commit) 2019-12-07T19:33:56.623+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 1 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.624247+0000 0 0.000000 2019-12-07T19:33:56.623+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 2 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2468 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2491 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2491 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e819a1b00 seq 1 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.fs1_head bits 4 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1738f~9bb to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1738f 0x1738f~9bb to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.fs1_head bits 4 = 0 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x738f 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x38f 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd4a and padding block with 0x2b6 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) get_onode oid 1#1:f0000000::::head# key 0x818000000000000001f000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) r -2 v.len 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1075 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._infover' <- _infover 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 6(3) info: 1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 6(3) info: 1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) pg 0x558e81075000 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 6(3) info: 1.2s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s) 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1b00 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1b00 onodes 0x558e81a03180 shared_blobs 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore.extentmap(0x558e81a03288) update 1#1:f0000000::::head# 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore.extentmap(0x558e81a03288) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.6 ruub 36.073596236s 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:f0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1b00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 prepare 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1b00 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 io_done 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.073596236s) 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.6(3) activated and committed 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 19 handle_pg_create_info new pg pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0:7._attach_pg 1.fs1 0x558e81a9b000 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 osd.0 19 identify_splits_and_merges 1.fs1 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0:7._prime_splits 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(4),6(3) upset 0(0),1(2),3(5),4(1),5(4),6(3) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 0(0) primary objects 0 missing 0 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.fs1 18 -> 19 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [3,0,5,4,1,6]/[3,0,5,4,1,6] -- 3/3 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(1) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(4) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 6(3) objects 0 missing 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.623+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 osd.0 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(4) missing= 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 6(3) missing= 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 3(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.074133487s int 16s) +create_info 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.3(0) 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 merge_log log((0'0,0'0], crt=0'0) from osd.3(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.074133487s int 16s) 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.002764194s -> ruub 36.071369293s 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.071369293s 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.623+0000 7fe2caa6c700 15 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:2 2019-12-07T19:33:56.623+0000 7fe2c8267700 5 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.624497+0000 3 0.000045 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7fe2c8267700 5 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7fe2c8267700 5 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x17d4a 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000285792s (0.000000080s flush + 0.000285712s kv commit) 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 694 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1377 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e8195ef00 seq 2 2019-12-07T19:33:56.623+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1400 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1400 < min_flush_size 524288 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:56.623+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195ef00 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195ef00 onodes shared_blobs 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195ef00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x17d4a~578 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x17d4a 0x17d4a~578 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 prepare 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x7d4a 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195ef00 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd4a 2019-12-07T19:33:56.623+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 io_done 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.623+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x2c2 and padding block with 0xd3e 2019-12-07T19:33:56.623+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1500,0x558e811fa300,0x558e8195e300> 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.623+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 kv_submitted 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1500 2019-12-07T19:33:56.623+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 finishing 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1500 onodes 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1500 done 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df900 q now empty 2019-12-07T19:33:56.623+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1500 [] 2019-12-07T19:33:56.623+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.623+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 kv_submitted 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa300 2019-12-07T19:33:56.623+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 finishing 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa300 onodes 0x558e81a02f00 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa300 done 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 kv_submitted 2019-12-07T19:33:56.623+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa300 [] 2019-12-07T19:33:56.623+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.623+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 kv_submitted 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e300 2019-12-07T19:33:56.623+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 finishing 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e300 onodes 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 done 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:33:56.623+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e300 [] 2019-12-07T19:33:56.623+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.623+0000 7fe2caa6c700 15 osd.0 19 enqueue_peering_evt 1.as3 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.623+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.623+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558e8193b000 2019-12-07T19:33:56.620+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.623+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:56.623+0000 7fe2cb26d700 15 osd.0 19 enqueue_peering_evt 1.1s1 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.623+0000 7fe2cb26d700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.623+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.as0 1.as0 1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558e81228700 con 0x558e818d7c00 2019-12-07T19:33:56.623+0000 7fe2cb26d700 15 osd.0 19 enqueue_peering_evt 1.9s3 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.623+0000 7fe2cb26d700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.623+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.623+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.623+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.623+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558e819e4000 2019-12-07T19:33:56.624+0000 7fe2d4a80700 3 osd.0 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.624+0000 7fe2d4a80700 10 osd.0 19 no new maps here, dropping 2019-12-07T19:33:56.624+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.624+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.624+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.624+0000 7fe2c9a6a700 15 osd.0 19 enqueue_peering_evt 1.4s1 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.624+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.624+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:33:56.624+0000 7fe2cb26d700 10 osd.0 19 send_incremental_map 18 -> 19 to 0x558e818d7000 v2:172.21.15.131:6812/37076 2019-12-07T19:33:56.624+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.624+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558e81938000 2019-12-07T19:33:56.624+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(19..19 src has 1..19) v4 -- 0x558e81908480 con 0x558e818d7000 2019-12-07T19:33:56.624+0000 7fe2cb26d700 10 osd.0 19 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 18 -> 19 (shared) 2019-12-07T19:33:56.624+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.624+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.624+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.1s0 1.1s0 1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558e81228e00 con 0x558e818d7000 2019-12-07T19:33:56.624+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:33:56.624+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7fe2c9a6a700 10 osd.0 19 send_incremental_map 18 -> 19 to 0x558e818fe000 v2:172.21.15.109:6826/36939 2019-12-07T19:33:56.624+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.624+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558e8192b000 2019-12-07T19:33:56.624+0000 7fe2c9a6a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(19..19 src has 1..19) v4 -- 0x558e81957b00 con 0x558e818fe000 2019-12-07T19:33:56.624+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.624+0000 7fe2c9a6a700 10 osd.0 19 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 18 -> 19 (shared) 2019-12-07T19:33:56.624+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.624+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:33:56.624+0000 7fe2c9a6a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_info2(1.4s0 1.4s0 1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558e81229500 con 0x558e818fe000 2019-12-07T19:33:56.624+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.9s0 1.9s0 1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558e81229c00 con 0x558e818d7c00 2019-12-07T19:33:56.624+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.624+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.624+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.624+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x182c2 2019-12-07T19:33:56.624+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.624+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.624+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.624+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.624+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000269022s (0.000000098s flush + 0.000268924s kv commit) 2019-12-07T19:33:56.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1102 < min_flush_size 524288 2019-12-07T19:33:56.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:56.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:33:56.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1125 < min_flush_size 524288 2019-12-07T19:33:56.624+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x182c2~465 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.624+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x182c2 0x182c2~465 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.624+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.624+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x82c2 2019-12-07T19:33:56.624+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x2c2 2019-12-07T19:33:56.624+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.624+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x727 and padding block with 0x8d9 2019-12-07T19:33:56.624+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 7 ==== pg_info2(1.2s0 1.2s0 1.2s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (unknown 0 0 0) 0x558e8122aa00 con 0x558e818d7000 2019-12-07T19:33:56.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1b00> 2019-12-07T19:33:56.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.624+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 kv_submitted 2019-12-07T19:33:56.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1b00 2019-12-07T19:33:56.624+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.2s0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(5) info: 1.2s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s) 2019-12-07T19:33:56.624+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(5) info: 1.2s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.624+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 finishing 2019-12-07T19:33:56.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1b00 onodes 0x558e81a03180 2019-12-07T19:33:56.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1b00 done 2019-12-07T19:33:56.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195ef00 kv_submitted 2019-12-07T19:33:56.624+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1b00 [] 2019-12-07T19:33:56.624+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info) prio 255 cost 10 e19) pg 0 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 no pg, should create on OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info) prio 255 cost 10 e19) 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 19 _make_pg 1.7s3 2019-12-07T19:33:56.624+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.7(unlocked)] enter Initial 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 pg_epoch: 18 pg[1.7(unlocked)] enter NotTrimming 2019-12-07T19:33:56.624+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 7 ==== pg_info2(1.2s0 1.2s0 1.2s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (unknown 0 0 0) 0x558e8122aa00 con 0x558e818fe400 2019-12-07T19:33:56.624+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 8 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (unknown 0 0 0) 0x558e817df200 con 0x558e818d7000 2019-12-07T19:33:56.624+0000 7fe2d4a80700 20 osd.0 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.624+0000 7fe2d4a80700 10 osd.0 19 do_waiters -- start 2019-12-07T19:33:56.624+0000 7fe2d4a80700 10 osd.0 19 do_waiters -- finish 2019-12-07T19:33:56.624+0000 7fe2d4a80700 20 osd.0 19 _dispatch 0x558e817df200 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.624+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.2s0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(1) info: 1.2s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s) 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach 1.7s3_head fresh osr 0x558e81966780 2019-12-07T19:33:56.624+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(1) info: 1.2s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.7s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role 3 up [1,6,4,0,7,2] acting [1,6,4,0,7,2] history ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18 past_intervals ([0,0] all_participants= intervals=) 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 20.079758017s + 0s) 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:33:56.624+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:56.624+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:33:56.624+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:33:56.625110+0000 0 0.000000 2019-12-07T19:33:56.624+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=0 crt=0'0 unknown mbc={}] set_last_peering_reset 18 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 18 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.624+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:33:56.625150+0000 1 0.000032 2019-12-07T19:33:56.624+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:33:56.624+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:33:56.624+0000 7fe2c8267700 1 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:33:56.624+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:33:56.625173+0000 0 0.000000 2019-12-07T19:33:56.624+0000 7fe2c8267700 5 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 18 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e8195fb00 seq 1 2019-12-07T19:33:56.624+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.7s3_head bits 4 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.7s3_head bits 4 = 0 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.7s3_head 0x558e8122c5a0) get_onode oid 3#1:e0000000::::head# key 0x838000000000000001e000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.7s3_head 0x558e8122c5a0) r -2 v.len 0 2019-12-07T19:33:56.624+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1076 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:56.624+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._infover' <- _infover 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:56.624+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:56.624+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195fb00 cost 5773 (1 ios * 4000 + 1773 bytes) 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195fb00 onodes 0x558e81a03400 shared_blobs 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore.extentmap(0x558e81a03508) update 3#1:e0000000::::head# 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore.extentmap(0x558e81a03508) update inline shard 2 bytes from 0 extents 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:e0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195fb00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 prepare 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195fb00 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 io_done 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 19 handle_pg_create_info new pg pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0:7._attach_pg 1.7s3 0x558e8122e000 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 19 identify_splits_and_merges 1.7s3 e18 to e19 pg_nums {18=16} 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0:7._prime_splits 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map: 19 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.7s3 18 -> 19 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 18 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:56.624+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] plpg_on_pool_change 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] hit_set_clear 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] agent_stop 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 19 advance_pg new pool opts old pool opts 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] handle_activate_map: 19 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 18 while current is 19 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:33:56.624+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 13 ==== pg_info2(1.2s0 1.2s0 1.2s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (unknown 0 0 0) 0x558e8122aa00 con 0x558e818d7c00 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 18 MLogRec from 1(0) log log((0'0,0'0], crt=0'0) pi ([0,0] all_participants= intervals=) pg_lease(ru 0s ub 36.075985074s int 16s) +create_info 2019-12-07T19:33:56.624+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.2s0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(2) info: 1.2s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s) 2019-12-07T19:33:56.624+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(2) info: 1.2s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.624+0000 7fe2d4a80700 3 osd.0 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.624+0000 7fe2d4a80700 10 osd.0 19 no new maps here, dropping 2019-12-07T19:33:56.624+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 6 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (unknown 0 0 0) 0x558e819b2240 con 0x558e818fe400 2019-12-07T19:33:56.624+0000 7fe2d4a80700 20 osd.0 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.624+0000 7fe2d4a80700 10 osd.0 19 do_waiters -- start 2019-12-07T19:33:56.624+0000 7fe2d4a80700 10 osd.0 19 do_waiters -- finish 2019-12-07T19:33:56.624+0000 7fe2d4a80700 20 osd.0 19 _dispatch 0x558e819b2240 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.624+0000 7fe2d4a80700 3 osd.0 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.624+0000 7fe2d4a80700 10 osd.0 19 no new maps here, dropping 2019-12-07T19:33:56.624+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 14 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (unknown 0 0 0) 0x558e81957200 con 0x558e818d7c00 2019-12-07T19:33:56.624+0000 7fe2d4a80700 20 osd.0 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.624+0000 7fe2d4a80700 10 osd.0 19 do_waiters -- start 2019-12-07T19:33:56.624+0000 7fe2d4a80700 10 osd.0 19 do_waiters -- finish 2019-12-07T19:33:56.624+0000 7fe2d4a80700 20 osd.0 19 _dispatch 0x558e81957200 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.624+0000 7fe2d4a80700 3 osd.0 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.624+0000 7fe2d4a80700 10 osd.0 19 no new maps here, dropping 2019-12-07T19:33:56.624+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 merge_log log((0'0,0'0], crt=0'0) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0 may_include_deletes = 1) changed=0 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] proc_lease pg_lease(ru 0s ub 36.075985074s int 16s) 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] peer_clock_delta_lb 0.002166438s -> ruub 36.073818636s 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] proc_lease readable_until_ub now 36.073818636s 2019-12-07T19:33:56.624+0000 7fe2c8267700 5 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] exit Started/Stray 2019-12-07T19:33:56.625591+0000 3 0.000046 2019-12-07T19:33:56.624+0000 7fe2c8267700 5 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive 2019-12-07T19:33:56.624+0000 7fe2c8267700 5 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: In ReplicaActive, about to call activate 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] activate - no missing, moving last_complete 0'0 -> 0'0 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: Activate Finished 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e8195e300 seq 2 2019-12-07T19:33:56.624+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e300 cost 5259 (1 ios * 4000 + 1259 bytes) 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e300 onodes shared_blobs 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 prepare 2019-12-07T19:33:56.624+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e300 2019-12-07T19:33:56.624+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 io_done 2019-12-07T19:33:56.625+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x18727 2019-12-07T19:33:56.625+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.625+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.625+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.001020767s (0.000000103s flush + 0.001020664s kv commit) 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:56.625+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1366 < min_flush_size 524288 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 2 2019-12-07T19:33:56.625+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2468 < min_flush_size 524288 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:56.625+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2491 < min_flush_size 524288 2019-12-07T19:33:56.625+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2491 < min_flush_size 524288 2019-12-07T19:33:56.625+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.625+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x18727~9bb to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.625+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x18727 0x18727~9bb to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195ef00> 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.625+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 kv_submitted 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x8727 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195ef00 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x727 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xe2 and padding block with 0xf1e 2019-12-07T19:33:56.625+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 finishing 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195ef00 onodes 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195ef00 done 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:33:56.625+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195ef00 [] 2019-12-07T19:33:56.625+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(5) info: 1.2s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(5) info: 1.2s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) pg 0x558e81075000 2019-12-07T19:33:56.625+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(5) info: 1.2s5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s) 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.6 ruub 36.073596236s 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.3 ruub 36.073596236s 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.073596236s) 2019-12-07T19:33:56.625+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.3(5) activated and committed 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(4),6(3) upset 0(0),1(2),3(5),4(1),5(4),6(3) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 0(0) primary objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(1) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(4) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 6(3) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(4) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 6(3) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x190e2 2019-12-07T19:33:56.625+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.625+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.625+0000 7fe2caa6c700 15 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:3 2019-12-07T19:33:56.625+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.625+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(1) info: 1.2s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000238932s (0.000000096s flush + 0.000238836s kv commit) 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(1) info: 1.2s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) pg 0x558e81075000 2019-12-07T19:33:56.625+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195fb00,0x558e8195e300> 2019-12-07T19:33:56.625+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(1) info: 1.2s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s) 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.625+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 kv_submitted 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195fb00 2019-12-07T19:33:56.625+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 finishing 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195fb00 onodes 0x558e81a03400 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 36.073596236s 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 done 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 kv_submitted 2019-12-07T19:33:56.625+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195fb00 [] 2019-12-07T19:33:56.625+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:33:56.625+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 kv_submitted 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e300 2019-12-07T19:33:56.625+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 finishing 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e300 onodes 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 done 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.6 ruub 36.073596236s 2019-12-07T19:33:56.625+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e300 [] 2019-12-07T19:33:56.625+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.625+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.3 ruub 36.073596236s 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.073596236s) 2019-12-07T19:33:56.625+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.4(1) activated and committed 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(4),6(3) upset 0(0),1(2),3(5),4(1),5(4),6(3) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 0(0) primary objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(1) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(4) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 6(3) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(4) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 6(3) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.625+0000 7fe2caa6c700 15 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:4 2019-12-07T19:33:56.625+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(2) info: 1.2s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(2) info: 1.2s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) pg 0x558e81075000 2019-12-07T19:33:56.625+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(2) info: 1.2s2( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s) 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 36.073596236s 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 36.073596236s 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.6 ruub 36.073596236s 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:33:56.625+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.3 ruub 36.073596236s 2019-12-07T19:33:56.625+0000 7fe2c8267700 15 osd.0 19 enqueue_peering_evt 1.fs1 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.625+0000 7fe2c8267700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 0s (sent 36.073596236s) 2019-12-07T19:33:56.625+0000 7fe2c8267700 15 osd.0 19 enqueue_peering_evt 1.7s3 epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.625+0000 7fe2c8267700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) 2019-12-07T19:33:56.625+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.1(2) activated and committed 2019-12-07T19:33:56.625+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.625+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.625+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.625+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558e81a9b000 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(4),6(3) upset 0(0),1(2),3(5),4(1),5(4),6(3) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:33:56.625+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 0(0) primary objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(1) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.625+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.625+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.625+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19) prio 255 cost 10 e19) pg 0x558e8122e000 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(4) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2c425f700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 ActivateCommitted from 19 processed at 19 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 6(3) objects 0 missing 0 2019-12-07T19:33:56.625+0000 7fe2c425f700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 unknown mbc={}] state: react 19 telling primary 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.625+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.625+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.7s0 1.7s0 1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558e80ff3800 con 0x558e818d7c00 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(4) missing= 0 2019-12-07T19:33:56.625+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 6(3) missing= 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.625+0000 7fe2caa6c700 15 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:5 2019-12-07T19:33:56.625+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:56.625+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.625+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.fs0 1.fs0 1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558e80ff3100 con 0x558e818d7000 2019-12-07T19:33:56.625+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.627+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 6 ==== pg_info2(1.ds5 1.ds5 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e81229500 con 0x558e818d6000 2019-12-07T19:33:56.627+0000 7fe2e753b700 15 osd.0 19 enqueue_peering_evt 1.ds5 epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s) 2019-12-07T19:33:56.627+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.627+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.627+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.627+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 7 ==== pg_info2(1.ds5 1.ds5 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e81229500 con 0x558e818d6000 2019-12-07T19:33:56.627+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.627+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e19) pg 0x558e819e7000 2019-12-07T19:33:56.627+0000 7fe2c9269700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s) 2019-12-07T19:33:56.627+0000 7fe2e753b700 15 osd.0 19 enqueue_peering_evt 1.ds5 epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s) 2019-12-07T19:33:56.627+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.627+0000 7fe2c9269700 20 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.627+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:33:56.627+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:56.627+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e819a1200 seq 3 2019-12-07T19:33:56.627+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:56.627+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:56.627+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:56.627+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.627+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes shared_blobs 2019-12-07T19:33:56.627+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[] released 0x[] 2019-12-07T19:33:56.627+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:33:56.627+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:33:56.627+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.627+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e19) pg 0x558e819e7000 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7fe2c5261700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 2(0) info: 1.ds0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s) 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x190e2~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x190e2 0x190e2~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x90e2 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xe2 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x4f8 and padding block with 0xb08 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.627+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:33:56.627+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e81a0bb00 seq 4 2019-12-07T19:33:56.627+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:56.627+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:56.627+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0bb00 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0bb00 onodes shared_blobs 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0bb00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.627+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 prepare 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0bb00 2019-12-07T19:33:56.627+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 io_done 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x194f8 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000106110s (0.000000101s flush + 0.000106009s kv commit) 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1200> 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.627+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.627+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0bb00 kv_queued 2019-12-07T19:33:56.627+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:33:56.627+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x194f8~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x194f8 0x194f8~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x94f8 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x4f8 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x945 and padding block with 0x6bb 2019-12-07T19:33:56.627+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x19945 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.627+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000110119s (0.000000105s flush + 0.000110014s kv commit) 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0bb00> 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.627+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 kv_submitted 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0bb00 2019-12-07T19:33:56.627+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 finishing 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0bb00 onodes 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0bb00 done 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:33:56.627+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0bb00 [] 2019-12-07T19:33:56.627+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.627+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.627+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:56.628+0000 7fe2e2ca2700 10 osd.0 19 tick_without_osd_lock 2019-12-07T19:33:56.628+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffcf000/0x40000000/0x1680000000, data 0xf0f6/0x2f000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:56.628+0000 7fe2e2ca2700 25 osd.0 19 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453695+0000 last_rx_front 2019-12-07T19:33:56.453704+0000 2019-12-07T19:33:56.628+0000 7fe2e2ca2700 25 osd.0 19 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453559+0000 last_rx_front 2019-12-07T19:33:56.453627+0000 2019-12-07T19:33:56.628+0000 7fe2e2ca2700 25 osd.0 19 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453575+0000 last_rx_front 2019-12-07T19:33:56.453596+0000 2019-12-07T19:33:56.628+0000 7fe2e2ca2700 25 osd.0 19 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453839+0000 last_rx_front 2019-12-07T19:33:56.453807+0000 2019-12-07T19:33:56.628+0000 7fe2e2ca2700 25 osd.0 19 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453763+0000 last_rx_front 2019-12-07T19:33:56.453773+0000 2019-12-07T19:33:56.628+0000 7fe2e2ca2700 25 osd.0 19 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453751+0000 last_rx_front 2019-12-07T19:33:56.453819+0000 2019-12-07T19:33:56.628+0000 7fe2e2ca2700 25 osd.0 19 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453789+0000 last_rx_front 2019-12-07T19:33:56.453733+0000 2019-12-07T19:33:56.628+0000 7fe2e2ca2700 20 osd.0 19 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:56.628+0000 7fe2e2ca2700 10 osd.0 19 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:56.628+0000 7fe2e2ca2700 20 osd.0 19 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:56.628+0000 7fe2e2ca2700 10 osd.0 19 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:56.628+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 15 ==== pg_info2(1.7s3 1.7s3 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e80ff3800 con 0x558e818d7c00 2019-12-07T19:33:56.628+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.7s3 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:33:56.628+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.628+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.628+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.628+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.628+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) pg 0x558e8122e000 2019-12-07T19:33:56.628+0000 7fe2c425f700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:33:56.628+0000 7fe2c425f700 20 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.628+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:33:56.628+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:56.628+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e81a0b800 seq 3 2019-12-07T19:33:56.628+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:56.628+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:56.628+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:56.628+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b800 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.628+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b800 onodes shared_blobs 2019-12-07T19:33:56.628+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b800 allocated 0x[] released 0x[] 2019-12-07T19:33:56.628+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 prepare 2019-12-07T19:33:56.628+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b800 2019-12-07T19:33:56.628+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 io_done 2019-12-07T19:33:56.628+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.628+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:56.628+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.628+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.628+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:56.628+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.628+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.628+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.628+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x19945~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.628+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x19945 0x19945~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.628+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.628+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x9945 2019-12-07T19:33:56.628+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x945 2019-12-07T19:33:56.628+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.628+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd5b and padding block with 0x2a5 2019-12-07T19:33:56.628+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x19d5b 2019-12-07T19:33:56.628+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.628+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.628+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.628+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.628+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000135247s (0.000000105s flush + 0.000135142s kv commit) 2019-12-07T19:33:56.628+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:56.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0b800> 2019-12-07T19:33:56.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.628+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 kv_submitted 2019-12-07T19:33:56.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b800 2019-12-07T19:33:56.628+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 finishing 2019-12-07T19:33:56.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b800 onodes 2019-12-07T19:33:56.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 done 2019-12-07T19:33:56.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:56.628+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b800 [] 2019-12-07T19:33:56.628+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.628+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.628+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.628+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 16 ==== pg_info2(1.7s3 1.7s3 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e80ff3800 con 0x558e818d7c00 2019-12-07T19:33:56.628+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.7s3 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:33:56.628+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.629+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.629+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.629+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.629+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e19) pg 0x558e8122e000 2019-12-07T19:33:56.629+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.7s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:33:56.629+0000 7fe2c8267700 20 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.629+0000 7fe2c8267700 20 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.629+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:33:56.629+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:56.629+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e811fa300 seq 4 2019-12-07T19:33:56.629+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:56.629+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:33:56.629+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:56.629+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:56.629+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa300 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.629+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa300 onodes shared_blobs 2019-12-07T19:33:56.629+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.629+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 prepare 2019-12-07T19:33:56.629+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa300 2019-12-07T19:33:56.629+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 io_done 2019-12-07T19:33:56.629+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:56.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.629+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:56.629+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.629+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.629+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.629+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x19d5b~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.629+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x19d5b 0x19d5b~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.629+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.629+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0x9d5b 2019-12-07T19:33:56.629+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd5b 2019-12-07T19:33:56.629+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.629+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x1a8 and padding block with 0xe58 2019-12-07T19:33:56.629+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1a1a8 2019-12-07T19:33:56.629+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.629+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.629+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.629+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000097207s (0.000000097s flush + 0.000097110s kv commit) 2019-12-07T19:33:56.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa300> 2019-12-07T19:33:56.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:56.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.629+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 kv_submitted 2019-12-07T19:33:56.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa300 2019-12-07T19:33:56.629+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 finishing 2019-12-07T19:33:56.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa300 onodes 2019-12-07T19:33:56.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa300 done 2019-12-07T19:33:56.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:56.629+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa300 [] 2019-12-07T19:33:56.629+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.629+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.629+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.636+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.687+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.703+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 4 ==== pg_info2(1.4s1 1.4s1 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e81229500 con 0x558e818fe000 2019-12-07T19:33:56.703+0000 7fe2e753b700 15 osd.0 19 enqueue_peering_evt 1.4s1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 7(0) info: 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s) 2019-12-07T19:33:56.703+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 7(0) info: 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.703+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.703+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 7(0) info: 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 7(0) info: 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s)) prio 255 cost 10 e19) pg 0x558e81938000 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 7(0) info: 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s) 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:33:56.704+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 5 ==== pg_info2(1.4s1 1.4s1 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e81229500 con 0x558e818fe000 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e811fa900 seq 3 2019-12-07T19:33:56.704+0000 7fe2e753b700 15 osd.0 19 enqueue_peering_evt 1.4s1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 7(0) info: 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s) 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:33:56.704+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 7(0) info: 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa900 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa900 onodes shared_blobs 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa900 allocated 0x[] released 0x[] 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 prepare 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa900 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 io_done 2019-12-07T19:33:56.704+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.704+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 7(0) info: 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.704+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.704+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 7(0) info: 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s)) prio 255 cost 10 e19) pg 0x558e81938000 2019-12-07T19:33:56.704+0000 7fe2c5a62700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 7(0) info: 1.4s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s) 2019-12-07T19:33:56.704+0000 7fe2c5a62700 20 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.704+0000 7fe2c5a62700 20 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.704+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:33:56.704+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:33:56.704+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e811fb200 seq 4 2019-12-07T19:33:56.704+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:33:56.704+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:33:56.704+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:56.704+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:33:56.704+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fb200 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.704+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fb200 onodes shared_blobs 2019-12-07T19:33:56.704+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fb200 allocated 0x[] released 0x[] 2019-12-07T19:33:56.704+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb200 prepare 2019-12-07T19:33:56.704+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fb200 2019-12-07T19:33:56.704+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb200 io_done 2019-12-07T19:33:56.704+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.704+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:56.704+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:56.704+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.704+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 2 2019-12-07T19:33:56.704+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2101 < min_flush_size 524288 2019-12-07T19:33:56.704+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:33:56.704+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2124 < min_flush_size 524288 2019-12-07T19:33:56.704+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2124 < min_flush_size 524288 2019-12-07T19:33:56.704+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.704+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1a1a8~84c to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.704+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1a1a8 0x1a1a8~84c to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.704+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.704+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xa1a8 2019-12-07T19:33:56.704+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x1a8 2019-12-07T19:33:56.704+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.704+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x9f4 and padding block with 0x60c 2019-12-07T19:33:56.704+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1a9f4 2019-12-07T19:33:56.704+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.704+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.704+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.704+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.704+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000185525s (0.000000146s flush + 0.000185379s kv commit) 2019-12-07T19:33:56.704+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:56.704+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.704+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa900,0x558e811fb200> 2019-12-07T19:33:56.704+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.704+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 kv_submitted 2019-12-07T19:33:56.704+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa900 2019-12-07T19:33:56.704+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 finishing 2019-12-07T19:33:56.704+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa900 onodes 2019-12-07T19:33:56.704+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa900 done 2019-12-07T19:33:56.704+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb200 kv_submitted 2019-12-07T19:33:56.704+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa900 [] 2019-12-07T19:33:56.704+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.704+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb200 kv_submitted 2019-12-07T19:33:56.704+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fb200 2019-12-07T19:33:56.704+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb200 finishing 2019-12-07T19:33:56.704+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fb200 onodes 2019-12-07T19:33:56.704+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb200 done 2019-12-07T19:33:56.704+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df900 q now empty 2019-12-07T19:33:56.704+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fb200 [] 2019-12-07T19:33:56.704+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.704+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.704+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.704+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:56.705+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 4 ==== pg_info2(1.2s0 1.2s0 1.2s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1028+0+0 (unknown 0 0 0) 0x558e81229500 con 0x558e818d6800 2019-12-07T19:33:56.705+0000 7fe2e753b700 15 osd.0 19 enqueue_peering_evt 1.2s0 epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(4) info: 1.2s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s) 2019-12-07T19:33:56.705+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(4) info: 1.2s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(4) info: 1.2s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(4) info: 1.2s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s)) prio 255 cost 10 e19) pg 0x558e81075000 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 5(4) info: 1.2s4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease_ack(ruub 36.073596236s) 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.4 ruub 36.073596236s 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.1 ruub 36.073596236s 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.6 ruub 36.073596236s 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.5 ruub 36.073596236s 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until peer osd.3 ruub 36.073596236s 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] recalc_readable_until readable_until[_ub] 36.073596236s (sent 36.073596236s) 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: peer osd.5(4) activated and committed 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(4),6(3) upset 0(0),1(2),3(5),4(1),5(4),6(3) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 0(0) primary objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(1) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(4) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 6(3) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(4) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 6(3) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.705+0000 7fe2c6a64700 15 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] publish_stats_to_osd 19:6 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] state: all_activated_and_committed 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(4),6(3) upset 0(0),1(2),3(5),4(1),5(4),6(3) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 0(0) primary objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 4(1) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 5(4) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats shard 6(3) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 5(4) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats missing shard 6(3) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.705+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 5 ==== osd_map(19..19 src has 1..19) v4 ==== 689+0+0 (unknown 0 0 0) 0x558e81222000 con 0x558e818d6800 2019-12-07T19:33:56.705+0000 7fe2d4a80700 20 osd.0 19 OSD::ms_dispatch: osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 creating+activating mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.705+0000 7fe2d4a80700 10 osd.0 19 do_waiters -- start 2019-12-07T19:33:56.705+0000 7fe2d4a80700 10 osd.0 19 do_waiters -- finish 2019-12-07T19:33:56.705+0000 7fe2d4a80700 20 osd.0 19 _dispatch 0x558e81222000 osd_map(19..19 src has 1..19) v4 2019-12-07T19:33:56.705+0000 7fe2d4a80700 3 osd.0 19 handle_osd_map epochs [19,19], i have 19, src has [1,19] 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] state: mnow 20.160773867s >= prior_readable_until_ub 0s 2019-12-07T19:33:56.705+0000 7fe2d4a80700 10 osd.0 19 no new maps here, dropping 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 19 send_pg_created 2019-12-07T19:33:56.705+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- osd_pg_created(1.2) v1 -- 0x558e819c43c0 con 0x558e80f39000 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] share_pg_info 2019-12-07T19:33:56.705+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558e81228000 con 0x558e818d7c00 2019-12-07T19:33:56.705+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558e8122aa00 con 0x558e818d7000 2019-12-07T19:33:56.705+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558e81228700 con 0x558e818fe400 2019-12-07T19:33:56.705+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s4 1.2s4 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558e8122b100 con 0x558e818d6800 2019-12-07T19:33:56.705+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- pg_info2(1.2s3 1.2s3 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 -- 0x558e8122b800 con 0x558e818ffc00 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(4),6(3) upset 0(0),1(2),3(5),4(1),5(4),6(3) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 0(0) primary objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 4(1) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 5(4) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 6(3) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 5(4) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 6(3) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.705+0000 7fe2c6a64700 15 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] publish_stats_to_osd 19:7 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] check_local 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] needs_recovery is recovered 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] needs_backfill does not need backfill 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] activate all replicas clean, no recovery 2019-12-07T19:33:56.705+0000 7fe2c6a64700 15 osd.0 19 enqueue_peering_evt 1.2s0 epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered) prio 255 cost 10 e19) 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(4),6(3) upset 0(0),1(2),3(5),4(1),5(4),6(3) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 0(0) primary objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 4(1) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 5(4) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 6(3) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 5(4) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 6(3) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.705+0000 7fe2c6a64700 15 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] publish_stats_to_osd 19:8 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] agent_stop 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e8195ef00 seq 4 2019-12-07T19:33:56.705+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195ef00 cost 5258 (1 ios * 4000 + 1258 bytes) 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195ef00 onodes shared_blobs 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195ef00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 prepare 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195ef00 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 io_done 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered) prio 255 cost 10 e19) pg 0x558e81075000 2019-12-07T19:33:56.705+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 AllReplicasRecovered 2019-12-07T19:33:56.705+0000 7fe2c6a64700 5 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] exit Started/Primary/Active/Activating 2019-12-07T19:33:56.706542+0000 9 0.004409 2019-12-07T19:33:56.705+0000 7fe2c6a64700 5 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] enter Started/Primary/Active/Recovered 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] needs_recovery is recovered 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(4),6(3) upset 0(0),1(2),3(5),4(1),5(4),6(3) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 0(0) primary objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 1(2) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 3(5) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 4(1) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 5(4) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats shard 6(3) objects 0 missing 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats object_location_counts {} 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:33:56.705+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:33:56.705+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:33:56.705+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1089 < min_flush_size 524288 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 5(4) missing= 0 2019-12-07T19:33:56.705+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:33:56.705+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1112 < min_flush_size 524288 2019-12-07T19:33:56.705+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1112 < min_flush_size 524288 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats missing shard 6(3) missing= 0 2019-12-07T19:33:56.705+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats degraded 0 2019-12-07T19:33:56.705+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1a9f4~458 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] update_calc_stats misplaced 0 2019-12-07T19:33:56.705+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1a9f4 0x1a9f4~458 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.705+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xa9f4 2019-12-07T19:33:56.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.705+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x9f4 2019-12-07T19:33:56.705+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.705+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xe4c and padding block with 0x1b4 2019-12-07T19:33:56.705+0000 7fe2c6a64700 15 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] publish_stats_to_osd 19:9 2019-12-07T19:33:56.705+0000 7fe2c6a64700 5 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] exit Started/Primary/Active/Recovered 2019-12-07T19:33:56.706665+0000 0 0.000000 2019-12-07T19:33:56.705+0000 7fe2c6a64700 5 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active mbc={}] enter Started/Primary/Active/Clean 2019-12-07T19:33:56.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] share_pg_info 2019-12-07T19:33:56.706+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x558e83c46000 con 0x558e818d7c00 2019-12-07T19:33:56.706+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x558e83c46700 con 0x558e818d7000 2019-12-07T19:33:56.706+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x558e83c46e00 con 0x558e818fe400 2019-12-07T19:33:56.706+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s4 1.2s4 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x558e83c47500 con 0x558e818d6800 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1ae4c 2019-12-07T19:33:56.706+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- pg_info2(1.2s3 1.2s3 1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 -- 0x558e83c47c00 con 0x558e818ffc00 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000165308s (0.000000082s flush + 0.000165226s kv commit) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 17 ==== pg_info2(1.es4 1.es4 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e80ff3800 con 0x558e818d7c00 2019-12-07T19:33:56.706+0000 7fe2c6a64700 15 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] publish_stats_to_osd 19:10 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195ef00> 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 kv_submitted 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195ef00 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] finish_recovery 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.es4 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s) 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 finishing 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195ef00 onodes 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] clear_recovery_state 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195ef00 done 2019-12-07T19:33:56.706+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195ef00 [] 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 18 ==== pg_info2(1.9s3 1.9s3 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e80ff3800 con 0x558e818d7c00 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e8195fb00 seq 5 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.9s3 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.706+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195fb00 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195fb00 onodes shared_blobs 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195fb00 allocated 0x[] released 0x[] 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 19 ==== pg_info2(1.as3 1.as3 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e80ff3800 con 0x558e818d7c00 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 prepare 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195fb00 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 io_done 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.as3 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s) 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e19) pg 0x558e8193b000 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s) 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e8195e300 seq 3 2019-12-07T19:33:56.706+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e300 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e300 onodes shared_blobs 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 prepare 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e300 2019-12-07T19:33:56.706+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 io_done 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.706+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2101 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2124 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2124 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1ae4c~84c to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1ae4c 0x1ae4c~84c to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xae4c 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xe4c 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x698 and padding block with 0x968 2019-12-07T19:33:56.706+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.706+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.706+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.706+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e19) pg 0x558e81a98000 2019-12-07T19:33:56.706+0000 7fe2c4a60700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s) 2019-12-07T19:33:56.706+0000 7fe2c4a60700 20 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.706+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:33:56.706+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:33:56.706+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e819a1200 seq 3 2019-12-07T19:33:56.706+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:56.706+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:56.706+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:56.706+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.706+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes shared_blobs 2019-12-07T19:33:56.706+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[] released 0x[] 2019-12-07T19:33:56.706+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:33:56.706+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:33:56.706+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1b698 2019-12-07T19:33:56.706+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000174291s (0.000000096s flush + 0.000174195s kv commit) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1b698~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1b698 0x1b698~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xb698 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x698 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xaae and padding block with 0x552 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195fb00,0x558e8195e300> 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 kv_submitted 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195fb00 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 finishing 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195fb00 onodes 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 done 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195fb00 [] 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 kv_submitted 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e300 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 finishing 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e300 onodes 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 done 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e300 [] 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.706+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) pg 0x558e8192b000 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 9 ==== pg_info2(1.1s1 1.1s1 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e80ff3800 con 0x558e818d7000 2019-12-07T19:33:56.706+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1baae 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.1s1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s) 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.706+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000187308s (0.000000066s flush + 0.000187242s kv commit) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:56.706+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1200> 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e81a0a600 seq 3 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:33:56.706+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 10 ==== pg_info2(1.1s1 1.1s1 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e83c46700 con 0x558e818d7000 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:33:56.706+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.706+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.706+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:56.706+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a600 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a600 onodes shared_blobs 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.1s1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s) 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a600 allocated 0x[] released 0x[] 2019-12-07T19:33:56.706+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.706+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 prepare 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a600 2019-12-07T19:33:56.706+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 io_done 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e19) pg 0x558e819e4000 2019-12-07T19:33:56.706+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.706+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s) 2019-12-07T19:33:56.706+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.706+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.706+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.706+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:33:56.706+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e81a0a900 seq 3 2019-12-07T19:33:56.706+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:56.706+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:33:56.706+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a900 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a900 onodes shared_blobs 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a900 allocated 0x[] released 0x[] 2019-12-07T19:33:56.706+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 prepare 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a900 2019-12-07T19:33:56.706+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 io_done 2019-12-07T19:33:56.706+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.706+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.706+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e19) pg 0x558e819e4000 2019-12-07T19:33:56.706+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.1s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s) 2019-12-07T19:33:56.706+0000 7fe2c7265700 20 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.706+0000 7fe2c7265700 20 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.706+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:33:56.706+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:33:56.706+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e811fa300 seq 4 2019-12-07T19:33:56.706+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:33:56.706+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:33:56.706+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:56.706+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:33:56.706+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa300 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.706+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa300 onodes shared_blobs 2019-12-07T19:33:56.706+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa300 allocated 0x[] released 0x[] 2019-12-07T19:33:56.706+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 prepare 2019-12-07T19:33:56.706+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa300 2019-12-07T19:33:56.706+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 io_done 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2046 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 2 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2069 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2069 < min_flush_size 524288 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1baae~815 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1baae 0x1baae~815 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xbaae 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xaae 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.706+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x2c3 and padding block with 0xd3d 2019-12-07T19:33:56.706+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1c2c3 2019-12-07T19:33:56.707+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.707+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.707+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.707+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.707+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000145187s (0.000000057s flush + 0.000145130s kv commit) 2019-12-07T19:33:56.707+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.707+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.707+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:33:56.707+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.707+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.707+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.707+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1c2c3~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.707+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1c2c3 0x1c2c3~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.707+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.707+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xc2c3 2019-12-07T19:33:56.707+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x2c3 2019-12-07T19:33:56.707+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.707+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x710 and padding block with 0x8f0 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0a600,0x558e81a0a900> 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.707+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 kv_submitted 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a600 2019-12-07T19:33:56.707+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 finishing 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a600 onodes 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a600 done 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:33:56.707+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a600 [] 2019-12-07T19:33:56.707+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.707+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 kv_submitted 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a900 2019-12-07T19:33:56.707+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 finishing 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a900 onodes 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a900 done 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa300 kv_submitted 2019-12-07T19:33:56.707+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a900 [] 2019-12-07T19:33:56.707+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.707+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.707+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1c710 2019-12-07T19:33:56.707+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.707+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.707+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.707+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.707+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000209569s (0.000000063s flush + 0.000209506s kv commit) 2019-12-07T19:33:56.707+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa300> 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.707+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 kv_submitted 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa300 2019-12-07T19:33:56.707+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 finishing 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa300 onodes 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa300 done 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:33:56.707+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa300 [] 2019-12-07T19:33:56.707+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.707+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.707+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] _finish_recovery 2019-12-07T19:33:56.707+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] purge_strays 2019-12-07T19:33:56.707+0000 7fe2caa6c700 20 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:56.707+0000 7fe2caa6c700 15 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] publish_stats_to_osd 19:11 2019-12-07T19:33:56.707+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 8 ==== pg_info2(1.bs3 1.bs3 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e83c46700 con 0x558e818fe400 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.bs3 epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s) 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.708+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e19) pg 0x558e81928000 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 9 ==== pg_info2(1.bs3 1.bs3 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e83c46e00 con 0x558e818fe400 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.bs3 epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s) 2019-12-07T19:33:56.708+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s) 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.708+0000 7fe2ca26b700 20 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.708+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:56.708+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e8195e600 seq 3 2019-12-07T19:33:56.708+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:56.708+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 20 ==== pg_info2(1.es4 1.es4 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e83c46e00 con 0x558e818d7c00 2019-12-07T19:33:56.708+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:56.708+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e600 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.708+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e600 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e600 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.es4 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s) 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.708+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 prepare 2019-12-07T19:33:56.708+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e600 2019-12-07T19:33:56.708+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 io_done 2019-12-07T19:33:56.708+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 21 ==== pg_info2(1.9s3 1.9s3 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e83c46000 con 0x558e818d7c00 2019-12-07T19:33:56.708+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:56.708+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.708+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.9s3 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:33:56.708+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e19) pg 0x558e81928000 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.708+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:56.708+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.708+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7fe2c6263700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 4(0) info: 1.bs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s) 2019-12-07T19:33:56.708+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1c710~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e19) pg 0x558e8192b000 2019-12-07T19:33:56.708+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1c710 0x1c710~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 22 ==== pg_info2(1.as3 1.as3 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e83c46000 con 0x558e818d7c00 2019-12-07T19:33:56.708+0000 7fe2c6263700 20 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.708+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.708+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xc710 2019-12-07T19:33:56.708+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x710 2019-12-07T19:33:56.708+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.708+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb26 and padding block with 0x4da 2019-12-07T19:33:56.708+0000 7fe2cb26d700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.9s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:33:56.708+0000 7fe2c6263700 20 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.as3 epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s) 2019-12-07T19:33:56.708+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.708+0000 7fe2cb26d700 20 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.708+0000 7fe2cb26d700 20 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.708+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:56.708+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:33:56.708+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e811fa900 seq 4 2019-12-07T19:33:56.708+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e811fb500 seq 4 2019-12-07T19:33:56.708+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:56.708+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:56.708+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:33:56.708+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:33:56.708+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:56.708+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:56.708+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:56.708+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fb500 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.708+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:56.708+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fb500 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fb500 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa900 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.708+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa900 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa900 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 prepare 2019-12-07T19:33:56.708+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.708+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fb500 2019-12-07T19:33:56.708+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 io_done 2019-12-07T19:33:56.708+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.708+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e19) pg 0x558e81a98000 2019-12-07T19:33:56.708+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 prepare 2019-12-07T19:33:56.708+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e19) pg 0x558e8193b000 2019-12-07T19:33:56.708+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa900 2019-12-07T19:33:56.708+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 io_done 2019-12-07T19:33:56.708+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7fe2c8a68700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.es0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s) 2019-12-07T19:33:56.708+0000 7fe2c6a64700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 1(0) info: 1.as0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s) 2019-12-07T19:33:56.708+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.708+0000 7fe2c8a68700 20 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.708+0000 7fe2c6a64700 20 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.708+0000 7fe2c8a68700 20 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.708+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:33:56.708+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:33:56.708+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:33:56.708+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e8123a000 seq 4 2019-12-07T19:33:56.708+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:56.708+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e83c56000 seq 4 2019-12-07T19:33:56.708+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:33:56.708+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:33:56.708+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:33:56.708+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:56.708+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:33:56.708+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:56.708+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:33:56.708+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8123a000 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.708+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8123a000 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56000 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.708+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8123a000 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56000 onodes shared_blobs 2019-12-07T19:33:56.708+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56000 allocated 0x[] released 0x[] 2019-12-07T19:33:56.708+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8123a000 prepare 2019-12-07T19:33:56.708+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 prepare 2019-12-07T19:33:56.708+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8123a000 2019-12-07T19:33:56.708+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56000 2019-12-07T19:33:56.708+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8123a000 io_done 2019-12-07T19:33:56.708+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 io_done 2019-12-07T19:33:56.708+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.708+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1cb26 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.001996689s (0.000000102s flush + 0.001996587s kv commit) 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e600> 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 kv_submitted 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e600 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 finishing 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e600 onodes 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e600 done 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa900 kv_queued 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e600 [] 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2156 < min_flush_size 524288 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3234 < min_flush_size 524288 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4312 < min_flush_size 524288 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4335 < min_flush_size 524288 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4335 < min_flush_size 524288 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1cb26~10ef to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1cb26 0x1cb26~10ef to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xcb26 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb26 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc15 and padding block with 0x3eb 2019-12-07T19:33:56.710+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.710+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1dc15 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.710+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 4 cleaned 0 in 0.000123638s (0.000000082s flush + 0.000123556s kv commit) 2019-12-07T19:33:56.710+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fb500,0x558e811fa900,0x558e8123a000,0x558e83c56000> 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 kv_submitted 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fb500 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 finishing 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fb500 onodes 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb500 done 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fb500 [] 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 kv_submitted 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa900 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 finishing 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa900 onodes 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa900 done 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa900 [] 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8123a000 kv_submitted 2019-12-07T19:33:56.710+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8123a000 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8123a000 finishing 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8123a000 onodes 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8123a000 done 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8123a000 [] 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 kv_submitted 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56000 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 finishing 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56000 onodes 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56000 done 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56000 [] 2019-12-07T19:33:56.710+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.710+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.710+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.710+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:56.710+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:56.710+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.710+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.710+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:56.710+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:56.711+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 11 ==== pg_info2(1.fs1 1.fs1 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e83c46000 con 0x558e818d7000 2019-12-07T19:33:56.711+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.fs1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s) 2019-12-07T19:33:56.711+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.711+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.711+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.711+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.711+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e19) pg 0x558e81a9b000 2019-12-07T19:33:56.711+0000 7fe2c425f700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=0/0 les/c/f=0/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s) 2019-12-07T19:33:56.711+0000 7fe2c425f700 20 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18 2019-12-07T19:33:56.711+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:33:56.711+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:56.711+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e81a0a600 seq 3 2019-12-07T19:33:56.711+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:56.711+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:56.711+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:56.711+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a600 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:33:56.711+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a600 onodes shared_blobs 2019-12-07T19:33:56.711+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a600 allocated 0x[] released 0x[] 2019-12-07T19:33:56.711+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 prepare 2019-12-07T19:33:56.711+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a600 2019-12-07T19:33:56.711+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 io_done 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:56.711+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1dc15~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1dc15 0x1dc15~416 to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xdc15 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc15 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x2b and padding block with 0xfd5 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1e02b 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000146650s (0.000000091s flush + 0.000146559s kv commit) 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0a600> 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.711+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 kv_submitted 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a600 2019-12-07T19:33:56.711+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 finishing 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a600 onodes 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a600 done 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:33:56.711+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a600 [] 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.711+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.711+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.711+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 12 ==== pg_info2(1.fs1 1.fs1 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e19/19) v1 ==== 1110+0+0 (unknown 0 0 0) 0x558e83c46000 con 0x558e818d7000 2019-12-07T19:33:56.711+0000 7fe2e6d3a700 15 osd.0 19 enqueue_peering_evt 1.fs1 epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s) 2019-12-07T19:33:56.711+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e19) 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e19) queued 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e19) pg 0x558e81a9b000 2019-12-07T19:33:56.711+0000 7fe2c8267700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/0 les/c/f=19/0/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 19 epoch_requested: 19 MInfoRec from 3(0) info: 1.fs0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s) 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] update_history clearing past_intervals 2019-12-07T19:33:56.711+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:33:56.711+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e811fa600 seq 4 2019-12-07T19:33:56.711+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:56.711+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:56.711+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa600 cost 5241 (1 ios * 4000 + 1241 bytes) 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa600 onodes shared_blobs 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa600 allocated 0x[] released 0x[] 2019-12-07T19:33:56.711+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 prepare 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa600 2019-12-07T19:33:56.711+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 io_done 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1078 < min_flush_size 524288 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1101 < min_flush_size 524288 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1e02b~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1e02b 0x1e02b~44d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xe02b 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x2b 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x478 and padding block with 0xb88 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1e478 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:56.711+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000079401s (0.000000079s flush + 0.000079322s kv commit) 2019-12-07T19:33:56.711+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa600> 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:56.711+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 kv_submitted 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa600 2019-12-07T19:33:56.711+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 finishing 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa600 onodes 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa600 done 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:33:56.711+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa600 [] 2019-12-07T19:33:56.711+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:56.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:56.711+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.711+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:56.722+0000 7fe2e4535700 10 osd.0 19 tick 2019-12-07T19:33:56.722+0000 7fe2e4535700 10 osd.0 19 do_waiters -- start 2019-12-07T19:33:56.722+0000 7fe2e4535700 10 osd.0 19 do_waiters -- finish 2019-12-07T19:33:56.722+0000 7fe2e4535700 20 osd.0 19 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:56.737+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.787+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.837+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.887+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.937+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:56.987+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.012+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:33:57.037+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.054+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.6 172.21.15.109:0/36937 5 ==== osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 0.000099360s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8800 con 0x558e802a4c00 2019-12-07T19:33:57.054+0000 7fe2e753b700 20 osd.0 19 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.709052156s,0.000099360s]) 2019-12-07T19:33:57.054+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509657308s delta_ub 0.000499460s) v5 -- 0x558e818ef400 con 0x558e802a4c00 2019-12-07T19:33:57.054+0000 7fe2e753b700 10 osd.0 19 maybe_share_map con 0x558e818ffc00 v2:172.21.15.109:6818/36937 map epoch 18 -> 19 (as per caller) 2019-12-07T19:33:57.054+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.6 172.21.15.109:0/36937 5 ==== osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509157848s delta_ub 0.000099360s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9000 con 0x558e8182cc00 2019-12-07T19:33:57.054+0000 7fe2e753b700 20 osd.0 19 handle_osd_ping new stamps hbstamp(osd.6 up_from 13 peer_clock_delta [18446744073.708974425s,0.000099360s]) 2019-12-07T19:33:57.054+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36937 -- osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.054652+0000/20.509157848s send_stamp 20.509735039s delta_ub 0.000577191s) v5 -- 0x558e818ef400 con 0x558e8182cc00 2019-12-07T19:33:57.072+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.7 172.21.15.109:0/36939 5 ==== osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.690600417s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e80264c00 con 0x558e81749000 2019-12-07T19:33:57.072+0000 7fe2e6d3a700 20 osd.0 19 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690046266s,18446744073.690600417s]) 2019-12-07T19:33:57.072+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.527407750s delta_ub 0.019505350s) v5 -- 0x558e818ee200 con 0x558e81749000 2019-12-07T19:33:57.072+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.7 172.21.15.109:0/36939 5 ==== osd_ping(ping e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.507902400s delta_ub 18446744073.690600417s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e80264c00 con 0x558e81748c00 2019-12-07T19:33:57.072+0000 7fe2e6d3a700 20 osd.0 19 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690005696s,18446744073.690600417s]) 2019-12-07T19:33:57.072+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e19 up_from 13 ping_stamp 2019-12-07T19:33:57.072463+0000/20.507902400s send_stamp 20.527448320s delta_ub 0.019545920s) v5 -- 0x558e818ef000 con 0x558e81748c00 2019-12-07T19:33:57.087+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.137+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.187+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.238+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.288+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.338+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.388+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.438+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.488+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.516+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:33:57.538+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.589+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 1 osrs, 3 txcs 2019-12-07T19:33:57.589+0000 7fe2dc28f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock osr 0x558e8107a500 2 ios pending 2019-12-07T19:33:57.589+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 10 0x2000~1000 2019-12-07T19:33:57.589+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x2000~1000 crc 1cd2c56e 2019-12-07T19:33:57.589+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 10 0x1b000~1000 2019-12-07T19:33:57.589+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x1b000~1000 crc 5f344e98 2019-12-07T19:33:57.589+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.589+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish osr 0x558e8107a500 2019-12-07T19:33:57.589+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish dequeueing 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 10 osd.0 19 tick_without_osd_lock 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ffcf000/0x40000000/0x1680000000, data 0xf0f6/0x2f000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 25 osd.0 19 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453695+0000 last_rx_front 2019-12-07T19:33:56.453704+0000 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 25 osd.0 19 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453559+0000 last_rx_front 2019-12-07T19:33:56.453627+0000 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 25 osd.0 19 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453575+0000 last_rx_front 2019-12-07T19:33:56.453596+0000 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 25 osd.0 19 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453839+0000 last_rx_front 2019-12-07T19:33:56.453807+0000 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 25 osd.0 19 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453763+0000 last_rx_front 2019-12-07T19:33:56.453773+0000 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 25 osd.0 19 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453751+0000 last_rx_front 2019-12-07T19:33:56.453819+0000 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 25 osd.0 19 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453789+0000 last_rx_front 2019-12-07T19:33:56.453733+0000 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 20 osd.0 19 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 20 osd.0 19 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 20 osd.0 19 scrub_load_below_threshold loadavg per cpu 0.065 < max 5 = yes 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 20 osd.0 19 sched_scrub load_is_low=1 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 10 osd.0 19 sched_scrub 1.2s0 scheduled at 2019-12-07T19:35:23.718605+0000 > 2019-12-07T19:33:57.633838+0000 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 20 osd.0 19 sched_scrub done 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 10 osd.0 19 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 20 osd.0 19 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:57.633+0000 7fe2e2ca2700 10 osd.0 19 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:57.639+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.651+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:33:57.689+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.720+0000 7fe2e4535700 10 osd.0 19 tick 2019-12-07T19:33:57.720+0000 7fe2e4535700 10 osd.0 19 do_waiters -- start 2019-12-07T19:33:57.720+0000 7fe2e4535700 10 osd.0 19 do_waiters -- finish 2019-12-07T19:33:57.720+0000 7fe2e4535700 20 osd.0 19 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:57.724+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:33:57.739+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.789+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.839+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.889+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.939+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:57.989+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.039+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.089+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.096+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e81985c00 0x558e80ef8b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.096+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e81985c00 0x558e80ef8b00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.096+0000 7fe2e7d3c700 10 osd.0 19 ms_handle_authentication new session 0x558e818e84e0 con 0x558e81985c00 entity client.admin addr 2019-12-07T19:33:58.096+0000 7fe2e7d3c700 10 osd.0 19 ms_handle_authentication session 0x558e818e84e0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.096+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/4040951113 conn(0x558e81985c00 0x558e80ef8b00 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=client.4346 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:58.096+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4346 172.21.15.131:0/4040951113 1 ==== command(tid 1: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558e817b45a0 con 0x558e81985c00 2019-12-07T19:33:58.096+0000 7fe2d959f700 20 osd.0 19 OSD::ms_dispatch: command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:58.096+0000 7fe2d959f700 10 osd.0 19 do_waiters -- start 2019-12-07T19:33:58.096+0000 7fe2d959f700 10 osd.0 19 do_waiters -- finish 2019-12-07T19:33:58.096+0000 7fe2d959f700 20 osd.0 19 _dispatch 0x558e817b45a0 command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:33:58.096+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/4040951113 -- command_reply(tid 1: 0 ) v1 -- 0x558e8022f340 con 0x558e81985c00 2019-12-07T19:33:58.118+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4346 172.21.15.131:0/4040951113 2 ==== command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558e817b4780 con 0x558e81985c00 2019-12-07T19:33:58.118+0000 7fe2d959f700 20 osd.0 19 OSD::ms_dispatch: command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:33:58.118+0000 7fe2d959f700 10 osd.0 19 do_waiters -- start 2019-12-07T19:33:58.118+0000 7fe2d959f700 10 osd.0 19 do_waiters -- finish 2019-12-07T19:33:58.118+0000 7fe2d959f700 20 osd.0 19 _dispatch 0x558e817b4780 command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:33:58.119+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/4040951113 -- command_reply(tid 2: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558e8022fc00 con 0x558e81985c00 2019-12-07T19:33:58.140+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.190+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.228+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:33:58.240+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.290+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.309+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/4040951113 conn(0x558e81985c00 msgr2=0x558e80ef8b00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 83 2019-12-07T19:33:58.309+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/4040951113 conn(0x558e81985c00 msgr2=0x558e80ef8b00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:33:58.309+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/4040951113 conn(0x558e81985c00 0x558e80ef8b00 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.309+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/4040951113 conn(0x558e81985c00 0x558e80ef8b00 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:33:58.309+0000 7fe2d959f700 2 osd.0 19 ms_handle_reset con 0x558e81985c00 session 0x558e818e84e0 2019-12-07T19:33:58.340+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.390+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.440+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.490+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.531+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:33:58.540+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.591+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 63904 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.625+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 17 ==== osd_map(20..21 src has 1..21) v4 ==== 1230+0+0 (crc 0 0 0) 0x558e81909d40 con 0x558e80f39000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 osd.0 19 OSD::ms_dispatch: osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 osd.0 19 do_waiters -- start 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 osd.0 19 do_waiters -- finish 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 osd.0 19 _dispatch 0x558e81909d40 osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.625+0000 7fe2d959f700 3 osd.0 19 handle_osd_map epochs [20,21], i have 19, src has [1,21] 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 osd.0 19 handle_osd_map got inc map for epoch 20 2019-12-07T19:33:58.625+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:34c0e826:::osdmap.19:0# 0x0~0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~140c size 0x140c (5132) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e818669a0 blob([0x2f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e811d1110 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81867110 blob([!~1000,0x30000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x558e81867c00 sbid 0x0)) need 0x1000~40c cache has 0x[1000~40c] 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:34c0e826:::osdmap.19:0# 0x0~140c = 5132 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 osd.0 19 add_map_bl 19 5132 bytes 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 osd.0 19 got_full_map 20, nothing requested 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 osd.0 19 handle_osd_map got inc map for epoch 21 2019-12-07T19:33:58.625+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3180e826:::osdmap.20:0# 0x0~0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3180e826:::osdmap.20:0# key 0x7f7fffffffffffffff3180e8'&!osdmap.20!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3180e826:::osdmap.20:0# 0x0~0 = -2 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 osd.0 19 got_full_map 21, nothing requested 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 osd.0 19 handle_osd_map pg_num_history pg_num_history(e21 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps purged_snaps {20={},21={}} 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 osd.0 19 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e21 [1,21] lci=[13,21]) 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e811fb200 seq 13 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:cd1b972f:::inc_osdmap.20:0# key 0x7f7fffffffffffffffcd1b97'/!inc_osdmap.20!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:58.625+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:cd1b972f:::inc_osdmap.20:0# 0x0~24d 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1077 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:cd1b972f:::inc_osdmap.20:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24d 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fb200 1 blobs 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x31000~1000/1000,1000,0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x31000~1000] 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81867810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81867ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81867810 blob([0x31000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81867ce0 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e81867810) get_ref 0x0~24d Blob(0x558e81867810 blob([0x31000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81867ce0 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e81867810) get_ref init 0x1000, 1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558e81867810 blob([0x31000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558e81867ce0 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81867cf8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x24d 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7288) dirty_range mark inline shard dirty 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:cd1b972f:::inc_osdmap.20:0# 0x0~24d = 0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3180e826:::osdmap.20:0# key 0x7f7fffffffffffffff3180e8'&!osdmap.20!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:58.625+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3180e826:::osdmap.20:0# 0x0~140c 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1078 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3180e826:::osdmap.20:0# 0x0~140c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~40c 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbf4 on front/back, now 0x0~1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fb200 2 blobs 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x32000~2000/1000,2000,0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x32000~2000] 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81867dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81867e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81867dc0 blob([0x32000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81867e30 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e81867dc0) get_ref 0x0~1000 Blob(0x558e81867dc0 blob([0x32000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81867e30 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e81867dc0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e81867dc0 blob([0x32000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e81867e30 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81867e48 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81867ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81867f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81867ea0 blob([!~1000,0x33000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81867f10 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e81867ea0) get_ref 0x1000~40c Blob(0x558e81867ea0 blob([!~1000,0x33000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e81867f10 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e81867ea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~40c: 0x1000~40c Blob(0x558e81867ea0 blob([!~1000,0x33000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x558e81867f10 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81867f28 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x140c 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7508) dirty_range mark inline shard dirty 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3180e826:::osdmap.20:0# 0x0~140c = 0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c21b972f:::inc_osdmap.21:0# key 0x7f7fffffffffffffffc21b97'/!inc_osdmap.21!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:58.625+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c21b972f:::inc_osdmap.21:0# 0x0~24d 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1079 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c21b972f:::inc_osdmap.21:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24d 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fb200 1 blobs 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x34000~1000/1000,1000,0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x34000~1000] 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194fea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194ff10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194fea0 blob([0x34000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194ff10 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e8194fea0) get_ref 0x0~24d Blob(0x558e8194fea0 blob([0x34000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194ff10 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e8194fea0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558e8194fea0 blob([0x34000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558e8194ff10 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e8194ff28 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x24d 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7788) dirty_range mark inline shard dirty 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c21b972f:::inc_osdmap.21:0# 0x0~24d = 0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3a80e826:::osdmap.21:0# key 0x7f7fffffffffffffff3a80e8'&!osdmap.21!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:58.625+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3a80e826:::osdmap.21:0# 0x0~140c 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1080 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3a80e826:::osdmap.21:0# 0x0~140c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~40c 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbf4 on front/back, now 0x0~1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fb200 2 blobs 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x35000~2000/1000,2000,0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x35000~2000] 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e8b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d0230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e8b60 blob([0x35000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d0230 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e817e8b60) get_ref 0x0~1000 Blob(0x558e817e8b60 blob([0x35000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d0230 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e817e8b60) get_ref init 0x1000, 1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e817e8b60 blob([0x35000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e811d0230 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e811d0248 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e811d03f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d0620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e811d03f0 blob([!~1000,0x36000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d0620 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e811d03f0) get_ref 0x1000~40c Blob(0x558e811d03f0 blob([!~1000,0x36000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e811d0620 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e811d03f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~40c: 0x1000~40c Blob(0x558e811d03f0 blob([!~1000,0x36000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x558e811d0620 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e811d0638 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x140c 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7a08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3a80e826:::osdmap.21:0# 0x0~140c = 0 2019-12-07T19:33:58.625+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fb200 0 blobs 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:33:58.625+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:58.625+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:58.625+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fb200 0 blobs 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fb200 cost 41203 (7 ios * 4000 + 13203 bytes) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fb200 onodes 0x558e80251b80,0x558e817b9180,0x558e819b7180,0x558e819b7400,0x558e819b7680,0x558e819b7900 shared_blobs 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7288) update #-1:cd1b972f:::inc_osdmap.20:0# 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:cd1b972f:::inc_osdmap.20:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7508) update #-1:3180e826:::osdmap.20:0# 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7508) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3180e826:::osdmap.20:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7788) update #-1:c21b972f:::inc_osdmap.21:0# 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7788) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c21b972f:::inc_osdmap.21:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7a08) update #-1:3a80e826:::osdmap.21:0# 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7a08) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3a80e826:::osdmap.21:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fb200 allocated 0x[31000~6000] released 0x[] 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 freelist allocate 0x31000~6000 2019-12-07T19:33:58.625+0000 7fe2d959f700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb200 prepare 2019-12-07T19:33:58.625+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e811fb200 2019-12-07T19:33:58.626+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb200 aio_wait 2019-12-07T19:33:58.626+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fb200 2019-12-07T19:33:58.626+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb200 io_done 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 7048 < min_flush_size 524288 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 9026 < min_flush_size 524288 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b7180 had 1 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b7400 had 1 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b7680 had 1 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b7900 had 1 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3860725760 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998762) bluestore 86 GiB free (0.959997), bluefs_ratio 0.039953 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 9085 < min_flush_size 524288 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 9085 < min_flush_size 524288 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1e478~237d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1e478 0x1e478~237d to file(ino 16 size 0x20000 mtime 2019-12-07T19:33:55.616569+0000 allocated 20000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000]) 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _allocate len 0x7f5 from 0 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _allocate len 0x7f5 from 1 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd100000 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd130000~10000/10000,10000,acd100000 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x30000 to include allocated 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 24 (was clean) 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd0f0000~10000 x_off 0xe478 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x478 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x7f5 and padding block with 0x80b 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x30000 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x207f5 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (24) on file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]), flushing log 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 24 len 0x3c crc 0x43ef2669) 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x17000~1000 to file(ino 1 size 0x17000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x17000 0x17000~1000 to file(ino 1 size 0x17000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x18000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x17000 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x18000 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:33:58.626+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000016 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 24 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 1 in 0.000428256s (0.000040693s flush + 0.000387563s kv commit) 2019-12-07T19:33:58.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fb200> 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <0x558e811de3c0> 2019-12-07T19:33:58.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb200 kv_submitted 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fb200 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e811fb200 osr 0x558e8107a500 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e000) prepare_write seq 11 0x1b000~1000 crc 1f5a496 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e000) prepare_write seq 11 0x2000~1000 crc 887c26f3 2019-12-07T19:33:58.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 deferred_cleanup 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c900 onodes 0x558e80251b80,0x558e817b9180,0x558e8191aa00,0x558e8191ac80 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e8088 in 0x558e80219340) _finish_write discard buffer(0x558e81171320 space 0x558e817e8088 0x0~1000 writing nocache) 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e81d8 in 0x558e80219340) _finish_write discard buffer(0x558e811713b0 space 0x558e817e81d8 0x0~1000 writing nocache) 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e82b8 in 0x558e80219340) _finish_write discard buffer(0x558e81171440 space 0x558e817e82b8 0x1000~1000 writing nocache) 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c900 done 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0000 deferred_cleanup 2019-12-07T19:33:58.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c900 [] 2019-12-07T19:33:58.626+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 deferred_cleanup 2019-12-07T19:33:58.626+0000 7fe2d7285700 10 osd.0 19 _committed_osd_maps 20..21 2019-12-07T19:33:58.626+0000 7fe2d7285700 10 osd.0 19 advance to epoch 20 (<= last 21 <= newest_map 21) 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0000 onodes 0x558e80251b80,0x558e817b9180,0x558e8191af00,0x558e8191b180 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e8408 in 0x558e80219340) _finish_write discard buffer(0x558e811718c0 space 0x558e817e8408 0x0~1000 writing nocache) 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e8558 in 0x558e80219340) _finish_write discard buffer(0x558e811714d0 space 0x558e817e8558 0x0~1000 writing nocache) 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e8638 in 0x558e80219340) _finish_write discard buffer(0x558e81171560 space 0x558e817e8638 0x1000~1000 writing nocache) 2019-12-07T19:33:58.626+0000 7fe2d7285700 10 osd.0 20 advance to epoch 21 (<= last 21 <= newest_map 21) 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0000 done 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c000 deferred_cleanup 2019-12-07T19:33:58.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0000 [] 2019-12-07T19:33:58.626+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.626+0000 7fe2d7285700 7 osd.0 21 consume_map version 21 2019-12-07T19:33:58.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 deferred_cleanup 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 21 identify_splits_and_merges 1.9s3 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c000 onodes 0x558e80251b80,0x558e817b9180,0x558e819b6c80,0x558e819b6f00 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 21 identify_splits_and_merges 1.1s1 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d1128 in 0x558e80219340) _finish_write discard buffer(0x558e81170900 space 0x558e811d1128 0x0~1000 writing nocache) 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 21 identify_splits_and_merges 1.as3 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 21 identify_splits_and_merges 1.2s0 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e818664e8 in 0x558e80219340) _finish_write discard buffer(0x558e81170870 space 0x558e818664e8 0x0~1000 writing nocache) 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 21 identify_splits_and_merges 1.bs3 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81867c18 in 0x558e80219340) _finish_write discard buffer(0x558e81170990 space 0x558e81867c18 0x1000~1000 writing nocache) 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 21 identify_splits_and_merges 1.4s1 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 21 identify_splits_and_merges 1.ds5 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c000 done 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 21 identify_splits_and_merges 1.es4 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb200 deferred_queued 2019-12-07T19:33:58.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c000 [] 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 21 identify_splits_and_merges 1.7s3 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.626+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 21 identify_splits_and_merges 1.fs1 e19 to e21 pg_nums {18=16} 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 21 prune_pg_created 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 21 prune_pg_created pruning 1.2 2019-12-07T19:33:58.626+0000 7fe2d7285700 10 osd.0:0.consume_map 21 (was 19) 2019-12-07T19:33:58.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.626+0000 7fe2d7285700 10 osd.0:1.consume_map 21 (was 19) 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:33:58.626+0000 7fe2d7285700 10 osd.0:2.consume_map 21 (was 19) 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:33:58.626+0000 7fe2d7285700 10 osd.0:3.consume_map 21 (was 19) 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 2019-12-07T19:33:58.626+0000 7fe2d7285700 10 osd.0:4.consume_map 21 (was 19) 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:33:58.626+0000 7fe2d7285700 10 osd.0:5.consume_map 21 (was 19) 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:33:58.626+0000 7fe2d7285700 10 osd.0:6.consume_map 21 (was 19) 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:33:58.626+0000 7fe2d7285700 10 osd.0:7.consume_map 21 (was 19) 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:33:58.626+0000 7fe2d7285700 10 osd.0 21 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:58.626+0000 7fe2d7285700 15 osd.0 21 enqueue_peering_evt 1.9s3 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.626+0000 7fe2d7285700 15 osd.0 21 enqueue_peering_evt 1.1s1 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.626+0000 7fe2d7285700 15 osd.0 21 enqueue_peering_evt 1.as3 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2d7285700 15 osd.0 21 enqueue_peering_evt 1.2s0 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2d7285700 15 osd.0 21 enqueue_peering_evt 1.bs3 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558e8192b000 2019-12-07T19:33:58.626+0000 7fe2d7285700 15 osd.0 21 enqueue_peering_evt 1.4s1 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.626+0000 7fe2d7285700 15 osd.0 21 enqueue_peering_evt 1.ds5 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.626+0000 7fe2d7285700 15 osd.0 21 enqueue_peering_evt 1.es4 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.626+0000 7fe2d7285700 15 osd.0 21 enqueue_peering_evt 1.7s3 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.9s3 19 -> 20 2019-12-07T19:33:58.626+0000 7fe2d7285700 15 osd.0 21 enqueue_peering_evt 1.fs1 epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.626+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) 2019-12-07T19:33:58.626+0000 7fe2d7285700 7 osd.0 21 activate_map version 21 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 20 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558e8122e000 2019-12-07T19:33:58.626+0000 7fe2cc26f700 20 osd.0 21 agent_entry empty queue 2019-12-07T19:33:58.626+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 20 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.626+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558e81938000 2019-12-07T19:33:58.626+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 20 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.7s3 19 -> 20 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.626+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2c5a62700 20 osd.0:4.update_pg_epoch 1.4s1 19 -> 20 2019-12-07T19:33:58.626+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.626+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 19 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:33:58.626+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558e81928000 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 osd.0 pg_epoch: 19 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 pg_epoch: 20 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 20 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 osd.0 pg_epoch: 20 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 pg_epoch: 20 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 20 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 osd.0 pg_epoch: 20 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.626+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:33:58.626+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 osd.0 pg_epoch: 20 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 20 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 20 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 pg_epoch: 20 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.9s3 20 -> 21 2019-12-07T19:33:58.626+0000 7fe2c5a62700 20 osd.0 pg_epoch: 20 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.626+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs3 19 -> 20 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 pg_epoch: 20 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.626+0000 7fe2c5a62700 20 osd.0 pg_epoch: 20 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.626+0000 7fe2c5a62700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 20 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 osd.0 pg_epoch: 19 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map [4,6,2,0,1,5]/[4,6,2,0,1,5] -- 4/4 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 osd.0 pg_epoch: 20 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 20 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.626+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.7s3 20 -> 21 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 osd.0 pg_epoch: 20 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.626+0000 7fe2c5a62700 20 osd.0:4.update_pg_epoch 1.4s1 20 -> 21 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 osd.0 pg_epoch: 20 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 20 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:33:58.626+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs3_head options 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 osd.0 pg_epoch: 20 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 pg_epoch: 21 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 osd.0 pg_epoch: 20 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 pg_epoch: 21 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.626+0000 7fe2ca26b700 20 osd.0 pg_epoch: 20 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.626+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:33:58.626+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.626+0000 7fe2ca26b700 20 osd.0 pg_epoch: 20 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.626+0000 7fe2ca26b700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.626+0000 7fe2c5a62700 20 osd.0 pg_epoch: 21 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 pg_epoch: 21 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:33:58.626+0000 7fe2c5a62700 20 osd.0 pg_epoch: 21 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.626+0000 7fe2c5a62700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 osd.0 pg_epoch: 20 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.626+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs3 20 -> 21 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 osd.0 pg_epoch: 20 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map [4,6,2,0,1,5]/[4,6,2,0,1,5] -- 4/4 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2c5a62700 20 osd.0 pg_epoch: 21 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558e819e4000 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:33:58.626+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.626+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs3_head options 2019-12-07T19:33:58.626+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.1s1 19 -> 20 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.626+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2ca26b700 20 osd.0 pg_epoch: 21 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.626+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.626+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 19 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [3,0,6,4,5,1]/[3,0,6,4,5,1] -- 3/3 2019-12-07T19:33:58.626+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558e81a98000 2019-12-07T19:33:58.626+0000 7fe2ca26b700 20 osd.0 pg_epoch: 21 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.626+0000 7fe2ca26b700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 20 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.626+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.fs1 19 -> 20 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 20 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.626+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:33:58.626+0000 7fe2c4a60700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 19 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [3,0,5,4,1,6]/[3,0,5,4,1,6] -- 3/3 2019-12-07T19:33:58.626+0000 7fe2c4a60700 20 osd.0:6.update_pg_epoch 1.es4 19 -> 20 2019-12-07T19:33:58.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 20 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.626+0000 7fe2ca26b700 20 osd.0 pg_epoch: 21 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 20 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 pg_epoch: 20 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.626+0000 7fe2c4a60700 10 osd.0 pg_epoch: 19 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,3,4,5,0,6]/[1,3,4,5,0,6] -- 1/1 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 20 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 pg_epoch: 20 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.626+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:33:58.626+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:33:58.626+0000 7fe2c7265700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.626+0000 7fe2c4a60700 10 osd.0 pg_epoch: 20 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 20 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7fe2c4a60700 10 osd.0 pg_epoch: 20 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:33:58.627+0000 7fe2c425f700 20 osd.0 pg_epoch: 20 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7fe2c7265700 10 osd.0 pg_epoch: 20 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.1s1 20 -> 21 2019-12-07T19:33:58.627+0000 7fe2c4a60700 10 osd.0 pg_epoch: 20 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7fe2c425f700 20 osd.0 pg_epoch: 20 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7fe2c425f700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7fe2c4a60700 20 osd.0 pg_epoch: 20 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7fe2c7265700 10 osd.0 pg_epoch: 20 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [3,0,6,4,5,1]/[3,0,6,4,5,1] -- 3/3 2019-12-07T19:33:58.627+0000 7fe2c4a60700 20 osd.0 pg_epoch: 20 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7fe2c4a60700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7fe2c425f700 10 osd.0 pg_epoch: 20 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.fs1 20 -> 21 2019-12-07T19:33:58.627+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:33:58.627+0000 7fe2c425f700 10 osd.0 pg_epoch: 20 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_advance_map [3,0,5,4,1,6]/[3,0,5,4,1,6] -- 3/3 2019-12-07T19:33:58.627+0000 7fe2c4a60700 10 osd.0 pg_epoch: 20 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7fe2c4a60700 20 osd.0:6.update_pg_epoch 1.es4 20 -> 21 2019-12-07T19:33:58.627+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7fe2c7265700 20 osd.0 pg_epoch: 21 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7fe2c4a60700 10 osd.0 pg_epoch: 20 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,3,4,5,0,6]/[1,3,4,5,0,6] -- 1/1 2019-12-07T19:33:58.627+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:33:58.627+0000 7fe2c7265700 20 osd.0 pg_epoch: 21 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7fe2c7265700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:33:58.627+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7fe2c425f700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7fe2c7265700 20 osd.0 pg_epoch: 21 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7fe2c4a60700 20 osd.0 pg_epoch: 21 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7fe2c4a60700 20 osd.0 pg_epoch: 21 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7fe2c4a60700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.627+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.626+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558e8193b000 2019-12-07T19:33:58.627+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.627+0000 7fe2c4a60700 20 osd.0 pg_epoch: 21 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.as3 19 -> 20 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,6,2,0,5,4]/[1,6,2,0,5,4] -- 1/1 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 20 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 20 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 20 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 pg_epoch: 20 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 pg_epoch: 20 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 20 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.as3 20 -> 21 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 20 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_advance_map [1,6,2,0,5,4]/[1,6,2,0,5,4] -- 1/1 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558e81075000 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_advance_map: 20 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.2s0 19 -> 20 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 19 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_advance_map [0,4,1,6,5,3]/[0,4,1,6,5,3] -- 0/0 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 20 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Active advmap 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 20 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Started advmap 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 20 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 20 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 pg_epoch: 20 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] hit_set_clear 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 pg_epoch: 20 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] agent_stop 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 20 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_advance_map: 21 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.2s0 20 -> 21 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 20 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_advance_map [0,4,1,6,5,3]/[0,4,1,6,5,3] -- 0/0 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Active advmap 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Started advmap 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.627+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] plpg_on_pool_change 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] hit_set_clear 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] agent_stop 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] handle_activate_map: 21 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] activate_map 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: Active: handling ActMap 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] Active: kicking snap trim 2019-12-07T19:33:58.627+0000 7fe2caa6c700 7 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] state: handle ActMap primary 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.627+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] publish_stats_to_osd 19:12 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.627+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:58.627+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.628+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.628+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) queued 2019-12-07T19:33:58.628+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.628+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 21 epoch_requested: 21 NullEvt) prio 255 cost 10 e21) pg 0x558e819e7000 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 20 2019-12-07T19:33:58.628+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.ds5 19 -> 20 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 osd.0 pg_epoch: 19 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 osd.0 pg_epoch: 20 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 osd.0 pg_epoch: 20 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.628+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 osd.0 pg_epoch: 20 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.628+0000 7fe2c9269700 20 osd.0 pg_epoch: 20 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.628+0000 7fe2c9269700 20 osd.0 pg_epoch: 20 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.628+0000 7fe2c9269700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 osd.0 pg_epoch: 20 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map: 21 2019-12-07T19:33:58.628+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.ds5 20 -> 21 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 osd.0 pg_epoch: 20 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] state: Started advmap 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:58.628+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:58.628+0000 7fe2c9269700 20 osd.0 pg_epoch: 21 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] hit_set_clear 2019-12-07T19:33:58.628+0000 7fe2c9269700 20 osd.0 pg_epoch: 21 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] agent_stop 2019-12-07T19:33:58.628+0000 7fe2c9269700 20 osd.0 21 advance_pg new pool opts old pool opts 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] handle_activate_map: 21 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] activate_map 2019-12-07T19:33:58.628+0000 7fe2c9269700 20 osd.0 pg_epoch: 21 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] activate_map: Not dirtying info: last_persisted is 19 while current is 21 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] do_peering_event: epoch_sent: 21 epoch_requested: 21 NullEvt 2019-12-07T19:33:58.628+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:33:58.628+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.629+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 23 ==== osd_map(20..21 src has 1..21) v4 ==== 1230+0+0 (unknown 0 0 0) 0x558e81908480 con 0x558e818d7c00 2019-12-07T19:33:58.629+0000 7fe2d4a80700 20 osd.0 21 OSD::ms_dispatch: osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.629+0000 7fe2d4a80700 10 osd.0 21 do_waiters -- start 2019-12-07T19:33:58.629+0000 7fe2d4a80700 10 osd.0 21 do_waiters -- finish 2019-12-07T19:33:58.629+0000 7fe2d4a80700 20 osd.0 21 _dispatch 0x558e81908480 osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.629+0000 7fe2d4a80700 3 osd.0 21 handle_osd_map epochs [20,21], i have 21, src has [1,21] 2019-12-07T19:33:58.629+0000 7fe2d4a80700 10 osd.0 21 no new maps here, dropping 2019-12-07T19:33:58.629+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 24 ==== MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 14256+0+0 (unknown 0 0 0) 0x558e819d9800 con 0x558e818d7c00 2019-12-07T19:33:58.629+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81942c60 prio 127 cost 0 latency 0.000053 epoch 21 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.629+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558e8192b000 2019-12-07T19:33:58.629+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 25 ==== MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:13, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 22448+0+0 (unknown 0 0 0) 0x558e819da000 con 0x558e818d7c00 2019-12-07T19:33:58.629+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81942dc0 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:13, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.629+0000 7fe2e6d3a700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:13, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e81942c60 prio 127 cost 0 latency 0.000097 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 osd.0 21 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 19 -> 21 (as per caller) 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:7, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:7 2019-12-07T19:33:58.626281+0000 0 ObjectCleanRegions clean_offsets: [49152~18446744073709502463], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:7 2019-12-07T19:33:58.626281+0000 0 ObjectCleanRegions clean_offsets: [49152~18446744073709502463], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.629+0000 7fe2cb26d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e811fb800 seq 5 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) get_onode oid 3#1:9694a0e5:::smithi13140796-3:head# key 0x8380000000000000019694a0e521736d'ithi13140796-3!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.629+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1081 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# = 0 2019-12-07T19:33:58.629+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 2 keys 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 2 keys = 0 2019-12-07T19:33:58.629+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.629+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x0~3000 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:9694a0e5:::smithi13140796-3:head# 0x0~3000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~3000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fb800 2 blobs 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 fbmap_alloc 0x558e80233300 allocate 0x3000/1000,3000,0 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x37000~3000/1000,3000,0 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x37000~3000] 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194e690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e8d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194e690 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8d20 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore.blob(0x558e8194e690) get_ref 0x0~2000 Blob(0x558e8194e690 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8d20 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore.blob(0x558e8194e690) get_ref init 0x2000, 1000 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e8194e690 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e817e8d38 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194f340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194f3b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194f340 blob([0x39000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f3b0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore.blob(0x558e8194f340) get_ref 0x0~1000 Blob(0x558e8194f340 blob([0x39000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f3b0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore.blob(0x558e8194f340) get_ref init 0x1000, 1000 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~1000: 0x0~1000 Blob(0x558e8194f340 blob([0x39000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194f3b0 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8194f3c8 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x3000 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e81a03c88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x0~3000 = 0 2019-12-07T19:33:58.629+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.629+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fb800 cost 26715 (3 ios * 4000 + 14715 bytes) 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fb800 onodes 0x558e81a03b80 shared_blobs 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e81a03c88) update 3#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e81a03c88) update inline shard 38 bytes from 2 extents 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9694a0e5:::smithi13140796-3:head# is 455 (411 bytes onode + 2 bytes spanning blobs + 42 bytes inline extents) 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fb800 allocated 0x[37000~3000] released 0x[] 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 freelist allocate 0x37000~3000 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 prepare 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e811fb800 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.629+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e81942c60 finish 2019-12-07T19:33:58.629+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:13, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:13, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558e81a98000 2019-12-07T19:33:58.629+0000 7fe2c8a68700 10 osd.0 21 dequeue_op 0x558e81942dc0 prio 127 cost 0 latency 0.000457 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:13, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.629+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:13, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.629+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:13 2019-12-07T19:33:58.627318+0000 0 ObjectCleanRegions clean_offsets: [81920~18446744073709469695], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.629+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:13 2019-12-07T19:33:58.627318+0000 0 ObjectCleanRegions clean_offsets: [81920~18446744073709469695], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.629+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.629+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.629+0000 7fe2c8a68700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.629+0000 7fe2c8a68700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.629+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e8123a300 seq 5 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.es4_head 0x558e819c4960) get_onode oid 4#1:733d7474:::smithi13140796-5:head# key 0x848000000000000001733d74't!smithi13140796-5!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.629+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1082 2019-12-07T19:33:58.629+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.es4_head 4#1:733d7474:::smithi13140796-5:head# = 0 2019-12-07T19:33:58.629+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2 keys 2019-12-07T19:33:58.629+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2 keys = 0 2019-12-07T19:33:58.629+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.es4_head 4#1:733d7474:::smithi13140796-5:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.629+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.es4_head 4#1:733d7474:::smithi13140796-5:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.629+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x0~5000 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 4#1:733d7474:::smithi13140796-5:head# 0x0~5000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.629+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~5000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8123a300 3 blobs 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.629+0000 7fe2c8a68700 10 fbmap_alloc 0x558e80233300 allocate 0x5000/1000,5000,0 2019-12-07T19:33:58.629+0000 7fe2c8a68700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x3a000~5000/1000,5000,0 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x3a000~5000] 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e8fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e9180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9180 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore.blob(0x558e817e8fc0) get_ref 0x0~2000 Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9180 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore.blob(0x558e817e8fc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9180 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e817e9198 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194e930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194ea10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194ea10 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore.blob(0x558e8194e930) get_ref 0x0~2000 Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194ea10 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore.blob(0x558e8194e930) get_ref init 0x2000, 1000 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ea10 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e8194ea28 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194ea80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194eb60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194ea80 blob([0x3e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194eb60 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore.blob(0x558e8194ea80) get_ref 0x0~1000 Blob(0x558e8194ea80 blob([0x3e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194eb60 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore.blob(0x558e8194ea80) get_ref init 0x1000, 1000 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~1000: 0x0~1000 Blob(0x558e8194ea80 blob([0x3e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194eb60 sbid 0x0)) 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e8194eb78 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:33:58.629+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x5000 2019-12-07T19:33:58.630+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.630+0000 7fe2c8a68700 20 bluestore.extentmap(0x558e811dcb08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.630+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x0~5000 = 0 2019-12-07T19:33:58.630+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.630+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.630+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:58.630+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.630+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:33:58.630+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.630+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.630+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.630+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:58.630+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8123a300 cost 38907 (4 ios * 4000 + 22907 bytes) 2019-12-07T19:33:58.630+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8123a300 onodes 0x558e811dca00 shared_blobs 2019-12-07T19:33:58.630+0000 7fe2c8a68700 20 bluestore.extentmap(0x558e811dcb08) update 4#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:33:58.630+0000 7fe2c8a68700 20 bluestore.extentmap(0x558e811dcb08) update inline shard 57 bytes from 3 extents 2019-12-07T19:33:58.630+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 4#1:733d7474:::smithi13140796-5:head# is 475 (412 bytes onode + 2 bytes spanning blobs + 61 bytes inline extents) 2019-12-07T19:33:58.630+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8123a300 allocated 0x[3a000~5000] released 0x[] 2019-12-07T19:33:58.630+0000 7fe2c8a68700 10 freelist allocate 0x3a000~5000 2019-12-07T19:33:58.630+0000 7fe2c8a68700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.630+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8123a300 prepare 2019-12-07T19:33:58.630+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8123a300 2019-12-07T19:33:58.630+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.630+0000 7fe2c8a68700 10 osd.0 21 dequeue_op 0x558e81942dc0 finish 2019-12-07T19:33:58.630+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.630+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 aio_wait 2019-12-07T19:33:58.630+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fb800 2019-12-07T19:33:58.630+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 io_done 2019-12-07T19:33:58.630+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8123a300 aio_wait 2019-12-07T19:33:58.630+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8123a300 2019-12-07T19:33:58.630+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8123a300 io_done 2019-12-07T19:33:58.630+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.630+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.630+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.630+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1979 < min_flush_size 524288 2019-12-07T19:33:58.630+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03b80 had 1 2019-12-07T19:33:58.630+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:33:58.630+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2002 < min_flush_size 524288 2019-12-07T19:33:58.630+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2002 < min_flush_size 524288 2019-12-07T19:33:58.630+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.630+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x207f5~7d2 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.630+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x207f5 0x207f5~7d2 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.630+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.630+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x7f5 2019-12-07T19:33:58.630+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x7f5 2019-12-07T19:33:58.630+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.630+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xfc7 and padding block with 0x39 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x20fc7 2019-12-07T19:33:58.631+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.631+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.631+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000558112s (0.000095219s flush + 0.000462893s kv commit) 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.631+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1999 < min_flush_size 524288 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dca00 had 1 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:33:58.631+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2022 < min_flush_size 524288 2019-12-07T19:33:58.631+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2022 < min_flush_size 524288 2019-12-07T19:33:58.631+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.631+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x20fc7~7e6 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.631+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x20fc7 0x20fc7~7e6 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xfc7 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xfc7 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x7ad and padding block with 0x853 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fb800> 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.631+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 kv_submitted 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fb800 2019-12-07T19:33:58.631+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 finishing 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fb800 onodes 0x558e81a03b80 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e8d38 in 0x558e80219a40) _finish_write discard buffer(0x558e811715f0 space 0x558e817e8d38 0x0~2000 writing nocache) 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194f3c8 in 0x558e80219a40) _finish_write discard buffer(0x558e81171050 space 0x558e8194f3c8 0x0~1000 writing nocache) 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb800 done 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:33:58.631+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fb800 [] 2019-12-07T19:33:58.631+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x217ad 2019-12-07T19:33:58.631+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.631+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.631+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000122641s (0.000001313s flush + 0.000121328s kv commit) 2019-12-07T19:33:58.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8123a300> 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.631+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8123a300 kv_submitted 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8123a300 2019-12-07T19:33:58.631+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8123a300 finishing 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8123a300 onodes 0x558e811dca00 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e9198 in 0x558e80219ea0) _finish_write discard buffer(0x558e811717a0 space 0x558e817e9198 0x0~2000 writing nocache) 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194ea28 in 0x558e80219ea0) _finish_write discard buffer(0x558e81171830 space 0x558e8194ea28 0x0~2000 writing nocache) 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194eb78 in 0x558e80219ea0) _finish_write discard buffer(0x558e811718c0 space 0x558e8194eb78 0x0~1000 writing nocache) 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8123a300 done 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:33:58.631+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.631+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8123a300 [] 2019-12-07T19:33:58.631+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.631+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e81955a00 con 0x558e818d7c00 2019-12-07T19:33:58.631+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.631+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.632+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e819dea00 con 0x558e818d7c00 2019-12-07T19:33:58.632+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.633+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 26 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 42932+0+0 (unknown 0 0 0) 0x558e819da800 con 0x558e818d7c00 2019-12-07T19:33:58.633+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81942f20 prio 127 cost 0 latency 0.000030 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.633+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.633+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 27 ==== MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 26544+0+0 (unknown 0 0 0) 0x558e811b8000 con 0x558e818d7c00 2019-12-07T19:33:58.633+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943080 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.633+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558e8192b000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e81943080 prio 127 cost 0 latency 0.000062 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81942f20 prio 127 cost 0 latency 0.000135 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:28, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:16, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:28 2019-12-07T19:33:58.631830+0000 0 ObjectCleanRegions clean_offsets: [163840~18446744073709387775], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [21'2 (0'0) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:16 2019-12-07T19:33:58.628073+0000 0 ObjectCleanRegions clean_offsets: [98304~18446744073709453311], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:28 2019-12-07T19:33:58.631830+0000 0 ObjectCleanRegions clean_offsets: [163840~18446744073709387775], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] add_log_entry 21'2 (0'0) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:16 2019-12-07T19:33:58.628073+0000 0 ObjectCleanRegions clean_offsets: [98304~18446744073709453311], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.633+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.633+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e81a0bb00 seq 6 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e81a0a300 seq 5 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) get_onode oid 3#1:9b8adabe:::smithi13140796-6:head# key 0x8380000000000000019b8adabe21736d'ithi13140796-6!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.7s3_head 0x558e8122c5a0) get_onode oid 3#1:ecc72bdb:::smithi13140796-10:head# key 0x838000000000000001ecc72bdb21736d'ithi13140796-10!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.633+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1083 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# = 0 2019-12-07T19:33:58.633+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1084 2019-12-07T19:33:58.633+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 2 keys 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# = 0 2019-12-07T19:33:58.633+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# 2 keys 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 2 keys = 0 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# 2 keys = 0 2019-12-07T19:33:58.633+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.633+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.633+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x0~6000 2019-12-07T19:33:58.633+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# 0x0~a000 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:9b8adabe:::smithi13140796-6:head# 0x0~6000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:ecc72bdb:::smithi13140796-10:head# 0x0~a000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~6000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~a000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e81a0bb00 3 blobs 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e81a0a300 5 blobs 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 fbmap_alloc 0x558e80233300 allocate 0x6000/1000,6000,0 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x3f000~6000/1000,6000,0 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x3f000~6000] 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194e2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e818667e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e818667e0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore.blob(0x558e8194e2a0) get_ref 0x0~2000 Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e818667e0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 fbmap_alloc 0x558e80233300 allocate 0xa000/1000,a000,0 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore.blob(0x558e8194e2a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x45000~a000/1000,a000,0 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x45000~a000] 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e818667f8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81867ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e9260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81867ab0 blob([0x45000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9260 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194e310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194e1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.blob(0x558e81867ab0) get_ref 0x0~2000 Blob(0x558e81867ab0 blob([0x45000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9260 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.blob(0x558e81867ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e1c0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e81867ab0 blob([0x45000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9260 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore.blob(0x558e8194e310) get_ref 0x0~2000 Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e1c0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e817e9278 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore.blob(0x558e8194e310) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8194e1d8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81866700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e9030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81866700 blob([0x47000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9030 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.blob(0x558e81866700) get_ref 0x0~2000 Blob(0x558e81866700 blob([0x47000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9030 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.blob(0x558e81866700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e81866700 blob([0x47000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9030 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e817e9048 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81866460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81867960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81867960 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore.blob(0x558e81866460) get_ref 0x0~2000 Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81867960 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore.blob(0x558e81866460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81866930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d00e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81866930 blob([0x49000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d00e0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e81867978 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.blob(0x558e81866930) get_ref 0x0~2000 Blob(0x558e81866930 blob([0x49000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d00e0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.blob(0x558e81866930) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e81866930 blob([0x49000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d00e0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e811d00f8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c64850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c647e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x6000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c64850 blob([0x4b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c647e0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.blob(0x558e83c64850) get_ref 0x0~2000 Blob(0x558e83c64850 blob([0x4b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c647e0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.blob(0x558e83c64850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83c64850 blob([0x4b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c647e0 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83c647f8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x0~6000 = 0 2019-12-07T19:33:58.633+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c64770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c64700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c64770 blob([0x4d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c64700 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.blob(0x558e83c64770) get_ref 0x0~2000 Blob(0x558e83c64770 blob([0x4d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c64700 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.blob(0x558e83c64770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83c64770 blob([0x4d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64700 sbid 0x0)) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83c64718 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0xa000 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83c72108) dirty_range mark inline shard dirty 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# 0x0~a000 = 0 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.633+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0bb00 cost 42195 (4 ios * 4000 + 26195 bytes) 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0bb00 onodes 0x558e811dd900 shared_blobs 2019-12-07T19:33:58.633+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) update 3#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) update inline shard 60 bytes from 3 extents 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9b8adabe:::smithi13140796-6:head# is 478 (412 bytes onode + 2 bytes spanning blobs + 64 bytes inline extents) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0bb00 allocated 0x[3f000~6000] released 0x[] 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 freelist allocate 0x3f000~6000 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a300 cost 67390 (6 ios * 4000 + 43390 bytes) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a300 onodes 0x558e83c72000 shared_blobs 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83c72108) update 3#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 prepare 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83c72108) update inline shard 98 bytes from 5 extents 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e81a0bb00 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:ecc72bdb:::smithi13140796-10:head# is 517 (413 bytes onode + 2 bytes spanning blobs + 102 bytes inline extents) 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a300 allocated 0x[45000~a000] released 0x[] 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 freelist allocate 0x45000~a000 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 prepare 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e81a0a300 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.633+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e81943080 finish 2019-12-07T19:33:58.633+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.633+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81942f20 finish 2019-12-07T19:33:58.633+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.634+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 aio_wait 2019-12-07T19:33:58.634+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0bb00 2019-12-07T19:33:58.634+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 io_done 2019-12-07T19:33:58.634+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 aio_wait 2019-12-07T19:33:58.634+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a300 2019-12-07T19:33:58.634+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 io_done 2019-12-07T19:33:58.634+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.634+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.634+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=2 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.635+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1188 < min_flush_size 524288 2019-12-07T19:33:58.635+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dd900 had 1 2019-12-07T19:33:58.635+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:33:58.635+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3231 < min_flush_size 524288 2019-12-07T19:33:58.635+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72000 had 1 2019-12-07T19:33:58.635+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.635+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3254 < min_flush_size 524288 2019-12-07T19:33:58.635+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3254 < min_flush_size 524288 2019-12-07T19:33:58.635+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.635+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x217ad~cb6 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.635+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x217ad 0x217ad~cb6 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.635+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.635+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x17ad 2019-12-07T19:33:58.635+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x7ad 2019-12-07T19:33:58.635+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.635+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x463 and padding block with 0xb9d 2019-12-07T19:33:58.635+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x22463 2019-12-07T19:33:58.635+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.635+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.635+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.635+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.635+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000592946s (0.000436320s flush + 0.000156626s kv commit) 2019-12-07T19:33:58.635+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0bb00,0x558e81a0a300> 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.635+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 kv_submitted 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0bb00 2019-12-07T19:33:58.635+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 finishing 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0bb00 onodes 0x558e811dd900 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e818667f8 in 0x558e80219a40) _finish_write discard buffer(0x558e81171560 space 0x558e818667f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81867978 in 0x558e80219a40) _finish_write discard buffer(0x558e81170990 space 0x558e81867978 0x0~2000 writing nocache) 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194e1d8 in 0x558e80219a40) _finish_write discard buffer(0x558e81171200 space 0x558e8194e1d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0bb00 done 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:33:58.635+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0bb00 [] 2019-12-07T19:33:58.635+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.635+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 kv_submitted 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a300 2019-12-07T19:33:58.635+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 finishing 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a300 onodes 0x558e83c72000 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d00f8 in 0x558e80219880) _finish_write discard buffer(0x558e81170870 space 0x558e811d00f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e9048 in 0x558e80219880) _finish_write discard buffer(0x558e81170900 space 0x558e817e9048 0x0~2000 writing nocache) 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e9278 in 0x558e80219880) _finish_write discard buffer(0x558e81171290 space 0x558e817e9278 0x0~2000 writing nocache) 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c64718 in 0x558e80219880) _finish_write discard buffer(0x558e81171050 space 0x558e83c64718 0x0~2000 writing nocache) 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c647f8 in 0x558e80219880) _finish_write discard buffer(0x558e811715f0 space 0x558e83c647f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a300 done 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.635+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a300 [] 2019-12-07T19:33:58.635+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.635+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.635+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.635+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=3, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e81238400 con 0x558e818d7c00 2019-12-07T19:33:58.635+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.635+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.635+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=4, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e818a8200 con 0x558e818d7c00 2019-12-07T19:33:58.635+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.640+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 28 ==== MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2228+0+0 (unknown 0 0 0) 0x558e819d9800 con 0x558e818d7c00 2019-12-07T19:33:58.640+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e819431e0 prio 127 cost 0 latency 0.000040 epoch 21 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.640+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.640+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.640+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.640+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e8192b000 2019-12-07T19:33:58.640+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e819431e0 prio 127 cost 0 latency 0.000091 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.9s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.640+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:8, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.640+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 29 ==== MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 55220+0+0 (unknown 0 0 0) 0x558e819d8800 con 0x558e818d7c00 2019-12-07T19:33:58.640+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943340 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.640+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.640+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'2], crt=0'0) [21'3 (21'1) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:8 2019-12-07T19:33:58.626309+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.640+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 21'2 active mbc={}] add_log_entry 21'3 (21'1) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:8 2019-12-07T19:33:58.626309+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.640+0000 7fe2c7265700 20 osd.0 pg_epoch: 21 pg[1.9s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:7 2019-12-07T19:33:58.626281+0000 0 ObjectCleanRegions clean_offsets: [49152~18446744073709502463], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.640+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.640+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.640+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.640+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.640+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:33:58.640+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e81a0b800 seq 7 2019-12-07T19:33:58.640+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 3 keys 2019-12-07T19:33:58.640+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 3 keys = 0 2019-12-07T19:33:58.640+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b800 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b800 onodes 0x558e81a03b80 shared_blobs 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.extentmap(0x558e81a03c88) update 3#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9694a0e5:::smithi13140796-3:head# is 526 (482 bytes onode + 2 bytes spanning blobs + 42 bytes inline extents) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 prepare 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b800 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 io_done 2019-12-07T19:33:58.641+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.641+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e819431e0 finish 2019-12-07T19:33:58.641+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1563 < min_flush_size 524288 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.641+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03b80 had 1 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e8192b000 2019-12-07T19:33:58.641+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:33:58.641+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1586 < min_flush_size 524288 2019-12-07T19:33:58.641+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1586 < min_flush_size 524288 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e81943340 prio 127 cost 0 latency 0.000215 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.9s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 lcod 21'2 active mbc={}] 2019-12-07T19:33:58.641+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 lcod 21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=7, reqid=client.4540.0:37, at_version=21'4, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.641+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x22463~632 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.641+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x22463 0x22463~632 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.641+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.641+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x2463 2019-12-07T19:33:58.641+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x463 2019-12-07T19:33:58.641+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 lcod 21'2 active mbc={}] append_log log((0'0,21'3], crt=21'1) [21'4 (0'0) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:37 2019-12-07T19:33:58.638297+0000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.641+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xa95 and padding block with 0x56b 2019-12-07T19:33:58.641+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'1 lcod 21'2 active mbc={}] add_log_entry 21'4 (0'0) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:37 2019-12-07T19:33:58.638297+0000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'1 lcod 21'2 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'1 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.641+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.641+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 78400 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e8106c900 seq 8 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) get_onode oid 3#1:9da56077:::smithi13140796-13:head# key 0x8380000000000000019da560'w!smithi13140796-13!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1085 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# = 0 2019-12-07T19:33:58.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 2 keys 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 2 keys = 0 2019-12-07T19:33:58.641+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x22a95 2019-12-07T19:33:58.641+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.641+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.641+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.641+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x0~d000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:9da56077:::smithi13140796-13:head# 0x0~d000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~d000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.641+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000138452s (0.000000210s flush + 0.000138242s kv commit) 2019-12-07T19:33:58.641+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0b800> 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c900 7 blobs 2019-12-07T19:33:58.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.641+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 kv_submitted 2019-12-07T19:33:58.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b800 2019-12-07T19:33:58.641+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 finishing 2019-12-07T19:33:58.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b800 onodes 0x558e81a03b80 2019-12-07T19:33:58.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 done 2019-12-07T19:33:58.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c900 prepare 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.641+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b800 [] 2019-12-07T19:33:58.641+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 fbmap_alloc 0x558e80233300 allocate 0xd000/1000,d000,0 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x4f000~d000/1000,d000,0 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x4f000~d000] 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c64e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c64d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c64d90 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83c64e70) get_ref 0x0~2000 Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c64d90 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83c64e70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83c64da8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81866e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194e0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81866e70 blob([0x51000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e0e0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.blob(0x558e81866e70) get_ref 0x0~2000 Blob(0x558e81866e70 blob([0x51000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e0e0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.blob(0x558e81866e70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e81866e70 blob([0x51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e0e0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8194e0f8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c642a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c641c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c642a0 blob([0x53000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c641c0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83c642a0) get_ref 0x0~2000 Blob(0x558e83c642a0 blob([0x53000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c641c0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83c642a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83c642a0 blob([0x53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c641c0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83c641d8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c64150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c65110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c64150 blob([0x55000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65110 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83c64150) get_ref 0x0~2000 Blob(0x558e83c64150 blob([0x55000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65110 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83c64150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83c64150 blob([0x55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65110 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83c65128 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c65180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c651f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c65180 blob([0x57000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c651f0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83c65180) get_ref 0x0~2000 Blob(0x558e83c65180 blob([0x57000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c651f0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83c65180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83c65180 blob([0x57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c651f0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83c65208 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c65260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c652d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c65260 blob([0x59000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c652d0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83c65260) get_ref 0x0~2000 Blob(0x558e83c65260 blob([0x59000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c652d0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83c65260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83c65260 blob([0x59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c652d0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83c652e8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c65340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c653b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c65340 blob([0x5b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c653b0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83c65340) get_ref 0x0~1000 Blob(0x558e83c65340 blob([0x5b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c653b0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83c65340) get_ref init 0x1000, 1000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~1000: 0x0~1000 Blob(0x558e83c65340 blob([0x5b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83c653b0 sbid 0x0)) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83c653c8 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0xd000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811ddc88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x0~d000 = 0 2019-12-07T19:33:58.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c900 cost 86870 (8 ios * 4000 + 54870 bytes) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c900 onodes 0x558e811ddb80 shared_blobs 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811ddc88) update 3#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811ddc88) update inline shard 133 bytes from 7 extents 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9da56077:::smithi13140796-13:head# is 552 (413 bytes onode + 2 bytes spanning blobs + 137 bytes inline extents) 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c900 allocated 0x[4f000~d000] released 0x[] 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 freelist allocate 0x4f000~d000 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 prepare 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8106c900 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'1 lcod 21'2 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.641+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e81943340 finish 2019-12-07T19:33:58.641+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.641+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=5, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e81955a00 con 0x558e818d7c00 2019-12-07T19:33:58.641+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.642+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 13 ==== osd_map(20..21 src has 1..21) v4 ==== 1230+0+0 (unknown 0 0 0) 0x558e81956900 con 0x558e818d7000 2019-12-07T19:33:58.642+0000 7fe2d4a80700 20 osd.0 21 OSD::ms_dispatch: osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.642+0000 7fe2d4a80700 10 osd.0 21 do_waiters -- start 2019-12-07T19:33:58.642+0000 7fe2d4a80700 10 osd.0 21 do_waiters -- finish 2019-12-07T19:33:58.642+0000 7fe2d4a80700 20 osd.0 21 _dispatch 0x558e81956900 osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.642+0000 7fe2d4a80700 3 osd.0 21 handle_osd_map epochs [20,21], i have 21, src has [1,21] 2019-12-07T19:33:58.642+0000 7fe2d4a80700 10 osd.0 21 no new maps here, dropping 2019-12-07T19:33:58.642+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 14 ==== MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:40, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 59316+0+0 (unknown 0 0 0) 0x558e819da000 con 0x558e818d7000 2019-12-07T19:33:58.642+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81942b00 prio 127 cost 0 latency 0.000041 epoch 21 MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:40, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.642+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:40, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.642+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 aio_wait 2019-12-07T19:33:58.642+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c900 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.642+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 io_done 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:40, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:40, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558e819e4000 2019-12-07T19:33:58.642+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.642+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.642+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e81942b00 prio 127 cost 0 latency 0.000151 MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:40, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 osd.0 21 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 19 -> 21 (as per caller) 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:40, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:40 2019-12-07T19:33:58.639869+0000 0 ObjectCleanRegions clean_offsets: [229376~18446744073709322239], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:40 2019-12-07T19:33:58.639869+0000 0 ObjectCleanRegions clean_offsets: [229376~18446744073709322239], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.642+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e8106c000 seq 5 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) get_onode oid 1#1:891f0d13:::smithi13140796-14:head# key 0x818000000000000001891f0d1321736d'ithi13140796-14!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.642+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1086 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# = 0 2019-12-07T19:33:58.642+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2 keys 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2 keys = 0 2019-12-07T19:33:58.642+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.642+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x0~e000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 1#1:891f0d13:::smithi13140796-14:head# 0x0~e000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~e000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c000 7 blobs 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 fbmap_alloc 0x558e80233300 allocate 0xe000/1000,e000,0 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x5c000~e000/1000,e000,0 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x5c000~e000] 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c65730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c64070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c65730 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c64070 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.blob(0x558e83c65730) get_ref 0x0~2000 Blob(0x558e83c65730 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c64070 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.blob(0x558e83c65730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83c65730 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83c64088 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c64e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c657a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c64e00 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c657a0 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.blob(0x558e83c64e00) get_ref 0x0~2000 Blob(0x558e83c64e00 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c657a0 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.blob(0x558e83c64e00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83c64e00 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83c657b8 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c65810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c658f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c65810 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c658f0 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.blob(0x558e83c65810) get_ref 0x0~2000 Blob(0x558e83c65810 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c658f0 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.blob(0x558e83c65810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83c65810 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83c65908 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e9c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e9b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e9c00 blob([0x62000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9b90 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.blob(0x558e817e9c00) get_ref 0x0~2000 Blob(0x558e817e9c00 blob([0x62000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9b90 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.blob(0x558e817e9c00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e817e9c00 blob([0x62000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e817e9ba8 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e9420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e9650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e9420 blob([0x64000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9650 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.blob(0x558e817e9420) get_ref 0x0~2000 Blob(0x558e817e9420 blob([0x64000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9650 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.blob(0x558e817e9420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e817e9420 blob([0x64000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e817e9668 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e95e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e96c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e95e0 blob([0x66000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e96c0 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.blob(0x558e817e95e0) get_ref 0x0~2000 Blob(0x558e817e95e0 blob([0x66000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e96c0 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.blob(0x558e817e95e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e817e95e0 blob([0x66000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e817e96d8 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e8850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e9c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e8850 blob([0x68000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9c70 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.blob(0x558e817e8850) get_ref 0x0~2000 Blob(0x558e817e8850 blob([0x68000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9c70 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.blob(0x558e817e8850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e817e8850 blob([0x68000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 sbid 0x0)) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e817e9c88 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0xe000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dd788) dirty_range mark inline shard dirty 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x0~e000 = 0 2019-12-07T19:33:58.642+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.642+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c000 cost 91774 (8 ios * 4000 + 59774 bytes) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c000 onodes 0x558e811dd680 shared_blobs 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dd788) update 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dd788) update inline shard 136 bytes from 7 extents 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:891f0d13:::smithi13140796-14:head# is 555 (413 bytes onode + 2 bytes spanning blobs + 140 bytes inline extents) 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c000 allocated 0x[5c000~e000] released 0x[] 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 freelist allocate 0x5c000~e000 2019-12-07T19:33:58.642+0000 7fe2c7265700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 prepare 2019-12-07T19:33:58.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8106c000 2019-12-07T19:33:58.642+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 30 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e811b8000 con 0x558e818d7c00 2019-12-07T19:33:58.642+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e819434a0 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.642+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.642+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.642+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.642+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.642+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.643+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e819434a0 prio 127 cost 0 latency 0.000069 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.7s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.643+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=9, reqid=client.4540.0:28, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.643+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.643+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:28 2019-12-07T19:33:58.631830+0000 0 ObjectCleanRegions clean_offsets: [163840~18446744073709387775], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.643+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.643+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.643+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.643+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.643+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.643+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e819a1200 seq 6 2019-12-07T19:33:58.643+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.643+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.643+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.643+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.643+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.643+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.643+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes shared_blobs 2019-12-07T19:33:58.643+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.643+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:33:58.643+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:33:58.643+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:33:58.643+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e819434a0 finish 2019-12-07T19:33:58.643+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.643+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 31 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e811b8800 con 0x558e818d7c00 2019-12-07T19:33:58.643+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943600 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.643+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1264 < min_flush_size 524288 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811ddb80 had 1 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1287 < min_flush_size 524288 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1287 < min_flush_size 524288 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x22a95~507 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x22a95 0x22a95~507 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x2a95 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xa95 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xf9c and padding block with 0x64 2019-12-07T19:33:58.643+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 32 ==== MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 2228+0+0 (unknown 0 0 0) 0x558e83ac8800 con 0x558e818d7c00 2019-12-07T19:33:58.643+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e818058c0 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x22f9c 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.643+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.643+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.643+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.643+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.643+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.643+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81943600 prio 127 cost 0 latency 0.000487 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.7s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.643+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:29, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.643+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:29 2019-12-07T19:33:58.631859+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.643+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:29 2019-12-07T19:33:58.631859+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.643+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.643+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.643+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.643+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.001555439s (0.001335308s flush + 0.000220131s kv commit) 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.643+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.643+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e81a0b500 seq 7 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.643+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# 3 keys 2019-12-07T19:33:58.643+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# 3 keys = 0 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.643+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.643+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.643+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x22f9c~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.643+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x22f9c 0x22f9c~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.643+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x2f9c 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xf9c 2019-12-07T19:33:58.643+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x40d and padding block with 0xbf3 2019-12-07T19:33:58.643+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.644+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.644+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.644+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.644+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b500 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.644+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b500 onodes 0x558e83c72000 shared_blobs 2019-12-07T19:33:58.644+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83c72108) update 3#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:33:58.644+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:ecc72bdb:::smithi13140796-10:head# is 589 (485 bytes onode + 2 bytes spanning blobs + 102 bytes inline extents) 2019-12-07T19:33:58.644+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.644+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 prepare 2019-12-07T19:33:58.644+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b500 2019-12-07T19:33:58.644+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 io_done 2019-12-07T19:33:58.644+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.644+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81943600 finish 2019-12-07T19:33:58.644+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e81942b00 finish 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558e8192b000 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e818058c0 prio 127 cost 0 latency 0.000302 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.9s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 lcod 21'3 active mbc={}] 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 lcod 21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:17, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 lcod 21'3 active mbc={}] append_log log((0'0,21'4], crt=21'1) [21'5 (21'2) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:17 2019-12-07T19:33:58.628090+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'1 lcod 21'3 active mbc={}] add_log_entry 21'5 (21'2) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:17 2019-12-07T19:33:58.628090+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] rollforward: entry=21'2 (0'0) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:16 2019-12-07T19:33:58.628073+0000 0 ObjectCleanRegions clean_offsets: [98304~18446744073709453311], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.644+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e819a0000 seq 9 2019-12-07T19:33:58.644+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 3 keys 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 3 keys = 0 2019-12-07T19:33:58.644+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.644+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0000 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0000 onodes 0x558e811dd900 shared_blobs 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) update 3#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9b8adabe:::smithi13140796-6:head# is 549 (483 bytes onode + 2 bytes spanning blobs + 64 bytes inline extents) 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 prepare 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0000 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 io_done 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'2 lcod 21'3 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.644+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e818058c0 finish 2019-12-07T19:33:58.644+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2340d 2019-12-07T19:33:58.644+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 aio_wait 2019-12-07T19:33:58.644+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.644+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c000 2019-12-07T19:33:58.644+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.644+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.644+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 io_done 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000330250s (0.000000174s flush + 0.000330076s kv commit) 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1629 < min_flush_size 524288 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72000 had 1 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3215 < min_flush_size 524288 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dd900 had 1 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:33:58.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5296 < min_flush_size 524288 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dd680 had 1 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:33:58.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5319 < min_flush_size 524288 2019-12-07T19:33:58.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5319 < min_flush_size 524288 2019-12-07T19:33:58.644+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2340d~14c7 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.644+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2340d 0x2340d~14c7 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x340d 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x40d 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x8d4 and padding block with 0x72c 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x248d4 2019-12-07T19:33:58.644+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.644+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.644+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 3 cleaned 0 in 0.000148266s (0.000003774s flush + 0.000144492s kv commit) 2019-12-07T19:33:58.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.644+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 33 ==== MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:43, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 63412+0+0 (unknown 0 0 0) 0x558e83ac9000 con 0x558e818d7c00 2019-12-07T19:33:58.644+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943760 prio 127 cost 0 latency 0.000031 epoch 21 MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:43, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.644+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:43, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:43, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:43, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558e8193b000 2019-12-07T19:33:58.644+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943760 prio 127 cost 0 latency 0.000089 MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:43, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.644+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:43, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.644+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:43 2019-12-07T19:33:58.641743+0000 0 ObjectCleanRegions clean_offsets: [245760~18446744073709305855], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.644+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:43 2019-12-07T19:33:58.641743+0000 0 ObjectCleanRegions clean_offsets: [245760~18446744073709305855], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.644+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.644+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.644+0000 7fe2c6a64700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.644+0000 7fe2c6a64700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.644+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e83c56300 seq 5 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.as3_head 0x558e819c5e00) get_onode oid 3#1:51ccf92f:::smithi13140796-15:head# key 0x83800000000000000151ccf9'/!smithi13140796-15!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.644+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1087 2019-12-07T19:33:58.644+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# = 0 2019-12-07T19:33:58.644+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# 2 keys 2019-12-07T19:33:58.644+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# 2 keys = 0 2019-12-07T19:33:58.644+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.644+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.644+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# 0x0~f000 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:51ccf92f:::smithi13140796-15:head# 0x0~f000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.644+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~f000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83c56300 8 blobs 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.644+0000 7fe2c6a64700 10 fbmap_alloc 0x558e80233300 allocate 0xf000/1000,f000,0 2019-12-07T19:33:58.644+0000 7fe2c6a64700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x6a000~f000/1000,f000,0 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x6a000~f000] 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e92d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c64f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e92d0 blob([0x6a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c64f50 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore.blob(0x558e817e92d0) get_ref 0x0~2000 Blob(0x558e817e92d0 blob([0x6a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c64f50 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore.blob(0x558e817e92d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e817e92d0 blob([0x6a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64f50 sbid 0x0)) 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83c64f68 in 0x558e80219b20) _discard 0x0~2000 2019-12-07T19:33:58.644+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e9880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e98f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e9880 blob([0x6c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e98f0 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.blob(0x558e817e9880) get_ref 0x0~2000 Blob(0x558e817e9880 blob([0x6c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e98f0 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.blob(0x558e817e9880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e817e9880 blob([0x6c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e98f0 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e817e9908 in 0x558e80219b20) _discard 0x0~2000 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e9960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e9a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e9960 blob([0x6e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9a40 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.blob(0x558e817e9960) get_ref 0x0~2000 Blob(0x558e817e9960 blob([0x6e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9a40 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.blob(0x558e817e9960) get_ref init 0x2000, 1000 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e817e9960 blob([0x6e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9a40 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e817e9a58 in 0x558e80219b20) _discard 0x0~2000 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e9ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c65f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e9ab0 blob([0x70000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65f80 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.blob(0x558e817e9ab0) get_ref 0x0~2000 Blob(0x558e817e9ab0 blob([0x70000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65f80 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.blob(0x558e817e9ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e817e9ab0 blob([0x70000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65f80 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83c65f98 in 0x558e80219b20) _discard 0x0~2000 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac4000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac4000 blob([0x72000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4070 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.blob(0x558e83ac4000) get_ref 0x0~2000 Blob(0x558e83ac4000 blob([0x72000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4070 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.blob(0x558e83ac4000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8106c900,0x558e819a1200,0x558e81a0b500,0x558e819a0000,0x558e8106c000> 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83ac4000 blob([0x72000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4070 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 kv_submitted 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83ac4088 in 0x558e80219b20) _discard 0x0~2000 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c900 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 finishing 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c900 onodes 0x558e811ddb80 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194e0f8 in 0x558e80219a40) _finish_write discard buffer(0x558e83c8e510 space 0x558e8194e0f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac40e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c641d8 in 0x558e80219a40) _finish_write discard buffer(0x558e83c8e5a0 space 0x558e83c641d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac40e0 blob([0x74000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4150 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c64da8 in 0x558e80219a40) _finish_write discard buffer(0x558e83c8e480 space 0x558e83c64da8 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.blob(0x558e83ac40e0) get_ref 0x0~2000 Blob(0x558e83ac40e0 blob([0x74000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4150 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.blob(0x558e83ac40e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c65128 in 0x558e80219a40) _finish_write discard buffer(0x558e83c8e630 space 0x558e83c65128 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c65208 in 0x558e80219a40) _finish_write discard buffer(0x558e83c8e6c0 space 0x558e83c65208 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83ac40e0 blob([0x74000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4150 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83ac4168 in 0x558e80219b20) _discard 0x0~2000 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c652e8 in 0x558e80219a40) _finish_write discard buffer(0x558e83c8e750 space 0x558e83c652e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c653c8 in 0x558e80219a40) _finish_write discard buffer(0x558e83c8e7e0 space 0x558e83c653c8 0x0~1000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c900 done 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0000 kv_submitted 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac41c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c900 [] 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac41c0 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4230 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.blob(0x558e83ac41c0) get_ref 0x0~2000 Blob(0x558e83ac41c0 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4230 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.blob(0x558e83ac41c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83ac41c0 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4230 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83ac4248 in 0x558e80219b20) _discard 0x0~2000 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b500 kv_submitted 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 kv_submitted 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b500 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 finishing 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac42a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b500 onodes 0x558e83c72000 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b500 done 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac42a0 blob([0x78000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4310 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b500 [] 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 kv_submitted 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.blob(0x558e83ac42a0) get_ref 0x0~1000 Blob(0x558e83ac42a0 blob([0x78000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4310 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0000 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.blob(0x558e83ac42a0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 finishing 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0000 onodes 0x558e811dd900 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0000 done 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~1000: 0x0~1000 Blob(0x558e83ac42a0 blob([0x78000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83ac4310 sbid 0x0)) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83ac4328 in 0x558e80219b20) _discard 0x0~1000 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0000 [] 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 kv_submitted 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c000 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0xf000 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 finishing 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c000 onodes 0x558e811dd680 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e9668 in 0x558e80219500) _finish_write discard buffer(0x558e83c8ebd0 space 0x558e817e9668 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83c72b08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e96d8 in 0x558e80219500) _finish_write discard buffer(0x558e83c8ec60 space 0x558e817e96d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e9ba8 in 0x558e80219500) _finish_write discard buffer(0x558e83c8eb40 space 0x558e817e9ba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# 0x0~f000 = 0 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e9c88 in 0x558e80219500) _finish_write discard buffer(0x558e83c8ecf0 space 0x558e817e9c88 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c64088 in 0x558e80219500) _finish_write discard buffer(0x558e83c8e990 space 0x558e83c64088 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c657b8 in 0x558e80219500) _finish_write discard buffer(0x558e83c8ea20 space 0x558e83c657b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c65908 in 0x558e80219500) _finish_write discard buffer(0x558e83c8eab0 space 0x558e83c65908 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c000 done 2019-12-07T19:33:58.645+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c000 [] 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.645+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.645+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:33:58.645+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56300 cost 99870 (9 ios * 4000 + 63870 bytes) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56300 onodes 0x558e83c72a00 shared_blobs 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83c72b08) update 3#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83c72b08) update inline shard 152 bytes from 8 extents 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:51ccf92f:::smithi13140796-15:head# is 571 (413 bytes onode + 2 bytes spanning blobs + 156 bytes inline extents) 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56300 allocated 0x[6a000~f000] released 0x[] 2019-12-07T19:33:58.645+0000 7fe2c6a64700 10 freelist allocate 0x6a000~f000 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:33:58.645+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 prepare 2019-12-07T19:33:58.645+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83c56300 2019-12-07T19:33:58.645+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=7, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558e818ee200 con 0x558e818d7c00 2019-12-07T19:33:58.645+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.645+0000 7fe2e2ca2700 10 osd.0 21 tick_without_osd_lock 2019-12-07T19:33:58.645+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=12, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558e818a8200 con 0x558e818d7c00 2019-12-07T19:33:58.645+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 21/18 ECSubWriteReply(tid=2, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e80264c00 con 0x558e818d7000 2019-12-07T19:33:58.645+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163ff87000/0x40000000/0x1680000000, data 0x53da8/0x77000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:58.645+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.645+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.645+0000 7fe2e2ca2700 25 osd.0 21 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453695+0000 last_rx_front 2019-12-07T19:33:56.453704+0000 2019-12-07T19:33:58.645+0000 7fe2e2ca2700 25 osd.0 21 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453559+0000 last_rx_front 2019-12-07T19:33:56.453627+0000 2019-12-07T19:33:58.645+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=9, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e81238600 con 0x558e818d7c00 2019-12-07T19:33:58.645+0000 7fe2e2ca2700 25 osd.0 21 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453575+0000 last_rx_front 2019-12-07T19:33:56.453596+0000 2019-12-07T19:33:58.645+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=10, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e819dee00 con 0x558e818d7c00 2019-12-07T19:33:58.645+0000 7fe2e2ca2700 25 osd.0 21 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453839+0000 last_rx_front 2019-12-07T19:33:56.453807+0000 2019-12-07T19:33:58.645+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.645+0000 7fe2e2ca2700 25 osd.0 21 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453763+0000 last_rx_front 2019-12-07T19:33:56.453773+0000 2019-12-07T19:33:58.645+0000 7fe2e2ca2700 25 osd.0 21 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453751+0000 last_rx_front 2019-12-07T19:33:56.453819+0000 2019-12-07T19:33:58.645+0000 7fe2e2ca2700 25 osd.0 21 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453789+0000 last_rx_front 2019-12-07T19:33:56.453733+0000 2019-12-07T19:33:58.645+0000 7fe2e2ca2700 20 osd.0 21 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:58.645+0000 7fe2e2ca2700 10 osd.0 21 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:58.645+0000 7fe2e2ca2700 20 osd.0 21 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:58.645+0000 7fe2e2ca2700 10 osd.0 21 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:58.645+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.645+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943760 finish 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.645+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 aio_wait 2019-12-07T19:33:58.645+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56300 2019-12-07T19:33:58.645+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558e818d7c00 msgr2=0x558e818e4000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 94 2019-12-07T19:33:58.645+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 io_done 2019-12-07T19:33:58.645+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558e818d7c00 msgr2=0x558e818e4000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.645+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558e818d7c00 0x558e818e4000 unknown :-1 s=READY pgs=6 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.645+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558e818d7c00 0x558e818e4000 unknown :-1 s=READY pgs=6 cs=0 l=0 rx=0 tx=0)._fault initiating reconnect 2019-12-07T19:33:58.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.645+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2097 < min_flush_size 524288 2019-12-07T19:33:58.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72a00 had 1 2019-12-07T19:33:58.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:33:58.645+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2120 < min_flush_size 524288 2019-12-07T19:33:58.645+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2120 < min_flush_size 524288 2019-12-07T19:33:58.645+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.645+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x248d4~848 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.645+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x248d4 0x248d4~848 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.645+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.645+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x48d4 2019-12-07T19:33:58.645+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x8d4 2019-12-07T19:33:58.645+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.645+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x11c and padding block with 0xee4 2019-12-07T19:33:58.645+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2511c 2019-12-07T19:33:58.645+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.645+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.645+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.645+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.645+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558e818d7c00 0x558e818e4000 unknown :-1 s=BANNER_CONNECTING pgs=6 cs=1 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000225414s (0.000116548s flush + 0.000108866s kv commit) 2019-12-07T19:33:58.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56300> 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 kv_submitted 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56300 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 finishing 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56300 onodes 0x558e83c72a00 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e9908 in 0x558e80219b20) _finish_write discard buffer(0x558e83c8f0e0 space 0x558e817e9908 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e9a58 in 0x558e80219b20) _finish_write discard buffer(0x558e811717a0 space 0x558e817e9a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac4088 in 0x558e80219b20) _finish_write discard buffer(0x558e83c8f200 space 0x558e83ac4088 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac4168 in 0x558e80219b20) _finish_write discard buffer(0x558e83c8f290 space 0x558e83ac4168 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac4248 in 0x558e80219b20) _finish_write discard buffer(0x558e81171830 space 0x558e83ac4248 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac4328 in 0x558e80219b20) _finish_write discard buffer(0x558e811718c0 space 0x558e83ac4328 0x0~1000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c64f68 in 0x558e80219b20) _finish_write discard buffer(0x558e81170b40 space 0x558e83c64f68 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c65f98 in 0x558e80219b20) _finish_write discard buffer(0x558e83c8f170 space 0x558e83c65f98 0x0~2000 writing nocache) 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56300 done 2019-12-07T19:33:58.645+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 21/18 ECSubWriteReply(tid=11, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e81965000 con 0x558e818d7c00 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56300 [] 2019-12-07T19:33:58.645+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.645+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.645+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.646+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558e818d7c00 0x558e818e4000 crc :-1 s=START_CONNECT pgs=6 cs=2 l=0 rx=0 tx=0)._fault waiting 15.000000 2019-12-07T19:33:58.646+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e80f39400 0x558e81986580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.646+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e80f39400 0x558e81986580 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.646+0000 7fe2e7d3c700 10 osd.0 21 ms_handle_authentication new session 0x558e818e8d00 con 0x558e80f39400 entity osd.1 addr 2019-12-07T19:33:58.646+0000 7fe2e7d3c700 10 osd.0 21 ms_handle_authentication session 0x558e818e8d00 osd.1 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.646+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558e80f39400 0x558e81986580 crc :-1 s=SESSION_ACCEPTING pgs=18 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x558e818d7c00 2019-12-07T19:33:58.646+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558e80f39400 0x558e81986580 crc :-1 s=SESSION_ACCEPTING pgs=18 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:58.646+0000 7fe2d4a80700 2 osd.0 21 ms_handle_reset con 0x558e80f39400 session 0x558e818e8d00 2019-12-07T19:33:58.646+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] conn(0x558e818d7c00 0x558e818e4000 unknown :-1 s=READY pgs=18 cs=3 l=0 rx=0 tx=0).ready entity=osd.1 client_cookie=1233dd104d1dac98 server_cookie=89d5462ffa9332cd in_seq=33 out_seq=23 2019-12-07T19:33:58.646+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 34 ==== MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=13, to_read={1:9694a0e5:::smithi13140796-3:head=0,4096,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[0,8]}, attrs_to_read=)) v3 ==== 239+0+0 (unknown 0 0 0) 0x558e819b6780 con 0x558e818d7c00 2019-12-07T19:33:58.646+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81805ce0 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=13, to_read={1:9694a0e5:::smithi13140796-3:head=0,4096,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.646+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=13, to_read={1:9694a0e5:::smithi13140796-3:head=0,4096,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.646+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 35 ==== MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=14, reqid=client.4540.0:13, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 909+0+0 (unknown 0 0 0) 0x558e819db800 con 0x558e818d7c00 2019-12-07T19:33:58.646+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81943080 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=14, reqid=client.4540.0:13, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.646+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=14, reqid=client.4540.0:13, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.646+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.646+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=13, to_read={1:9694a0e5:::smithi13140796-3:head=0,4096,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.646+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.646+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=13, to_read={1:9694a0e5:::smithi13140796-3:head=0,4096,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e8192b000 2019-12-07T19:33:58.646+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e81805ce0 prio 127 cost 0 latency 0.000108 MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=13, to_read={1:9694a0e5:::smithi13140796-3:head=0,4096,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.646+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=13, to_read={1:9694a0e5:::smithi13140796-3:head=0,4096,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.646+0000 7fe2c7265700 25 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.646+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x0~1000 2019-12-07T19:33:58.646+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x3000 (12288) 2019-12-07T19:33:58.646+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.646+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e690 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.646+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194e690 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.646+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.647+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.647+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 36 ==== MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=15, reqid=client.4540.0:14, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2228+0+0 (unknown 0 0 0) 0x558e811b8000 con 0x558e818d7c00 2019-12-07T19:33:58.647+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e819431e0 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=15, reqid=client.4540.0:14, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.647+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=15, reqid=client.4540.0:14, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.647+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194e690 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.647+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e817e8d38 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:33:58.647+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e81170f30 space 0x558e817e8d38 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.647+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x0~1000 = 4096 2019-12-07T19:33:58.647+0000 7fe2c7265700 20 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.647+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: Getting attr on 1:9694a0e5:::smithi13140796-3:head 2019-12-07T19:33:58.647+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: not in cache 1:9694a0e5:::smithi13140796-3:head 2019-12-07T19:33:58.647+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:33:58.647+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: found on disk, size 12288 2019-12-07T19:33:58.647+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# hinfo_key 2019-12-07T19:33:58.647+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# hinfo_key = 0 2019-12-07T19:33:58.647+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 21/18 ECSubReadReply(tid=13, attrs_read=0)) v2 -- 0x558e811dd400 con 0x558e818d7c00 2019-12-07T19:33:58.647+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e81805ce0 finish 2019-12-07T19:33:58.647+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.647+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.647+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=14, reqid=client.4540.0:13, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.647+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.647+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=14, reqid=client.4540.0:13, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e81a98000 2019-12-07T19:33:58.647+0000 7fe2c4a60700 10 osd.0 21 dequeue_op 0x558e81943080 prio 127 cost 0 latency 0.001005 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=14, reqid=client.4540.0:13, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.es4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.647+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=14, reqid=client.4540.0:13, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.647+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.647+0000 7fe2c4a60700 20 osd.0 pg_epoch: 21 pg[1.es4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:13 2019-12-07T19:33:58.627318+0000 0 ObjectCleanRegions clean_offsets: [81920~18446744073709469695], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.647+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.647+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.647+0000 7fe2c4a60700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.647+0000 7fe2c4a60700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.647+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:33:58.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e8195ef00 seq 6 2019-12-07T19:33:58.647+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:58.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.647+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:58.647+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195ef00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195ef00 onodes shared_blobs 2019-12-07T19:33:58.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195ef00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.647+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 prepare 2019-12-07T19:33:58.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195ef00 2019-12-07T19:33:58.647+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 io_done 2019-12-07T19:33:58.647+0000 7fe2c4a60700 10 osd.0 21 dequeue_op 0x558e81943080 finish 2019-12-07T19:33:58.647+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=15, reqid=client.4540.0:14, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=15, reqid=client.4540.0:14, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e81a98000 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 osd.0 21 dequeue_op 0x558e819431e0 prio 127 cost 0 latency 0.000751 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=15, reqid=client.4540.0:14, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.es4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2511c~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=15, reqid=client.4540.0:14, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2511c 0x2511c~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x511c 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x11c 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x58d and padding block with 0xa73 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:14 2019-12-07T19:33:58.627337+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:14 2019-12-07T19:33:58.627337+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.648+0000 7fe2c4a60700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2558d 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e8195fb00 seq 7 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000089272s (0.000000110s flush + 0.000089162s kv commit) 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195ef00> 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.648+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 kv_submitted 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195ef00 2019-12-07T19:33:58.648+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 3 keys 2019-12-07T19:33:58.648+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 finishing 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195ef00 onodes 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195ef00 done 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 prepare 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 3 keys = 0 2019-12-07T19:33:58.648+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195ef00 [] 2019-12-07T19:33:58.648+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.648+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.648+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195fb00 cost 5879 (1 ios * 4000 + 1879 bytes) 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195fb00 onodes 0x558e811dca00 shared_blobs 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 bluestore.extentmap(0x558e811dcb08) update 4#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 4#1:733d7474:::smithi13140796-5:head# is 546 (483 bytes onode + 2 bytes spanning blobs + 61 bytes inline extents) 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195fb00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 prepare 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195fb00 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 io_done 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1583 < min_flush_size 524288 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.648+0000 7fe2c4a60700 10 osd.0 21 dequeue_op 0x558e819431e0 finish 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dca00 had 1 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1606 < min_flush_size 524288 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1606 < min_flush_size 524288 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2558d~646 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2558d 0x2558d~646 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x558d 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x58d 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xbd3 and padding block with 0x42d 2019-12-07T19:33:58.648+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=14, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e818ef000 con 0x558e818d7c00 2019-12-07T19:33:58.648+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x25bd3 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.648+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000086745s (0.000000063s flush + 0.000086682s kv commit) 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195fb00> 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.648+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 kv_submitted 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195fb00 2019-12-07T19:33:58.648+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 finishing 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195fb00 onodes 0x558e811dca00 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 done 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:33:58.648+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195fb00 [] 2019-12-07T19:33:58.648+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.648+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.648+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=15, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e81955e00 con 0x558e818d7c00 2019-12-07T19:33:58.648+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.651+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 37 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e811b8800 con 0x558e818d7c00 2019-12-07T19:33:58.651+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81943340 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.651+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.651+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.3 172.21.15.131:0/37076 1 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:58.652090+0000/22.109602748s send_stamp 22.109602748s delta_ub 0.003028961s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8800 con 0x558e818ff800 2019-12-07T19:33:58.651+0000 7fe2e753b700 20 osd.0 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002719646s,0.003028961s]) 2019-12-07T19:33:58.651+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:58.652090+0000/22.109602748s send_stamp 22.106883102s delta_ub 18446744073.706831970s) v5 -- 0x558e818ef400 con 0x558e818ff800 2019-12-07T19:33:58.651+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.651+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.651+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.651+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.651+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81943340 prio 127 cost 0 latency 0.000329 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=16, reqid=client.4540.0:29, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.651+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:ecc72bdb:::smithi13140796-10:head by client.4540.0:29 2019-12-07T19:33:58.631859+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.651+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.651+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.651+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.651+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e81a0bb00 seq 8 2019-12-07T19:33:58.651+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.651+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.651+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.651+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.651+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.651+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0bb00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.651+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0bb00 onodes shared_blobs 2019-12-07T19:33:58.651+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0bb00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.651+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 prepare 2019-12-07T19:33:58.651+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0bb00 2019-12-07T19:33:58.651+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 io_done 2019-12-07T19:33:58.651+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81943340 finish 2019-12-07T19:33:58.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.651+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.651+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x25bd3~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.651+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x25bd3 0x25bd3~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.651+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.651+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x5bd3 2019-12-07T19:33:58.651+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xbd3 2019-12-07T19:33:58.651+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.651+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x44 and padding block with 0xfbc 2019-12-07T19:33:58.651+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 38 ==== MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=17, to_read={1:ecc72bdb:::smithi13140796-10:head=0,4096,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e811dd400 con 0x558e818d7c00 2019-12-07T19:33:58.651+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81943e40 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=17, to_read={1:ecc72bdb:::smithi13140796-10:head=0,4096,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.651+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=17, to_read={1:ecc72bdb:::smithi13140796-10:head=0,4096,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.651+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.651+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=17, to_read={1:ecc72bdb:::smithi13140796-10:head=0,4096,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.651+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.651+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=17, to_read={1:ecc72bdb:::smithi13140796-10:head=0,4096,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.651+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x26044 2019-12-07T19:33:58.651+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.651+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.651+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.652+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.651+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81943e40 prio 127 cost 0 latency 0.000052 MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=17, to_read={1:ecc72bdb:::smithi13140796-10:head=0,4096,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000137708s (0.000000133s flush + 0.000137575s kv commit) 2019-12-07T19:33:58.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.652+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=17, to_read={1:ecc72bdb:::smithi13140796-10:head=0,4096,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0bb00> 2019-12-07T19:33:58.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 kv_submitted 2019-12-07T19:33:58.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0bb00 2019-12-07T19:33:58.652+0000 7fe2c8267700 25 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 finishing 2019-12-07T19:33:58.652+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# 0x0~1000 2019-12-07T19:33:58.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0bb00 onodes 2019-12-07T19:33:58.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0bb00 done 2019-12-07T19:33:58.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0bb00 [] 2019-12-07T19:33:58.652+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0xa000 (40960) 2019-12-07T19:33:58.652+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.652+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.652+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81867ab0 blob([0x45000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9260 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.652+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81867ab0 blob([0x45000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9260 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.652+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.652+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.652+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.3 172.21.15.131:0/37076 1 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:58.652090+0000/22.109602748s send_stamp 22.109602748s delta_ub 0.003028961s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81239600 con 0x558e81984000 2019-12-07T19:33:58.652+0000 7fe2e6d3a700 20 osd.0 21 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002140647s,0.003028961s]) 2019-12-07T19:33:58.652+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:58.652090+0000/22.109602748s send_stamp 22.107462101s delta_ub 18446744073.707410969s) v5 -- 0x558e818ee800 con 0x558e81984000 2019-12-07T19:33:58.652+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.652+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81867ab0 blob([0x45000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9260 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.652+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e817e9278 in 0x558e80219880) _discard 0x0~1000 2019-12-07T19:33:58.652+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83c8e2d0 space 0x558e817e9278 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.652+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# 0x0~1000 = 4096 2019-12-07T19:33:58.652+0000 7fe2c8267700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.652+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: Getting attr on 1:ecc72bdb:::smithi13140796-10:head 2019-12-07T19:33:58.652+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: not in cache 1:ecc72bdb:::smithi13140796-10:head 2019-12-07T19:33:58.652+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:33:58.652+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: found on disk, size 40960 2019-12-07T19:33:58.652+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# hinfo_key 2019-12-07T19:33:58.652+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# hinfo_key = 0 2019-12-07T19:33:58.652+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 21/18 ECSubReadReply(tid=17, attrs_read=0)) v2 -- 0x558e81a03680 con 0x558e818d7c00 2019-12-07T19:33:58.652+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81943e40 finish 2019-12-07T19:33:58.652+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=16, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e81238400 con 0x558e818d7c00 2019-12-07T19:33:58.652+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.654+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 39 ==== MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 909+0+0 (unknown 0 0 0) 0x558e819da800 con 0x558e818d7c00 2019-12-07T19:33:58.655+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81942f20 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.655+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.655+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 40 ==== MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=19, reqid=client.4540.0:14, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 909+0+0 (unknown 0 0 0) 0x558e819d9800 con 0x558e818d7c00 2019-12-07T19:33:58.655+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e819434a0 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=19, reqid=client.4540.0:14, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.655+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=19, reqid=client.4540.0:14, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.655+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.655+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=19, reqid=client.4540.0:14, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.655+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.655+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=19, reqid=client.4540.0:14, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558e81a98000 2019-12-07T19:33:58.655+0000 7fe2c4a60700 10 osd.0 21 dequeue_op 0x558e819434a0 prio 127 cost 0 latency 0.000126 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=19, reqid=client.4540.0:14, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.es4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.655+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=19, reqid=client.4540.0:14, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.655+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.655+0000 7fe2c4a60700 20 osd.0 pg_epoch: 21 pg[1.es4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:733d7474:::smithi13140796-5:head by client.4540.0:14 2019-12-07T19:33:58.627337+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.655+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.655+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.655+0000 7fe2c4a60700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.655+0000 7fe2c4a60700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.655+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:33:58.655+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e819a1200 seq 8 2019-12-07T19:33:58.655+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:58.655+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.655+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.655+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.655+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:58.655+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.655+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes shared_blobs 2019-12-07T19:33:58.655+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.655+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:33:58.655+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:33:58.655+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:33:58.655+0000 7fe2c4a60700 10 osd.0 21 dequeue_op 0x558e819434a0 finish 2019-12-07T19:33:58.655+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.656+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 41 ==== MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=22, to_read={1:9b8adabe:::smithi13140796-6:head=0,4096,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[0,8]}, attrs_to_read=)) v3 ==== 239+0+0 (unknown 0 0 0) 0x558e81a03680 con 0x558e818d7c00 2019-12-07T19:33:58.656+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81942dc0 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=22, to_read={1:9b8adabe:::smithi13140796-6:head=0,4096,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.656+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=22, to_read={1:9b8adabe:::smithi13140796-6:head=0,4096,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.656+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 42 ==== MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e819d8800 con 0x558e818d7c00 2019-12-07T19:33:58.656+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81943e40 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.656+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558e8192b000 2019-12-07T19:33:58.656+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e81942f20 prio 127 cost 0 latency 0.001561 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.656+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=18, reqid=client.4540.0:17, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.656+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'5], crt=21'2) [] 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'3 (21'1) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:8 2019-12-07T19:33:58.626309+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'4 (0'0) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:37 2019-12-07T19:33:58.638297+0000 0 ObjectCleanRegions clean_offsets: [212992~18446744073709338623], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'5 (21'2) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:17 2019-12-07T19:33:58.628090+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.656+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.656+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.656+0000 7fe2cb26d700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.656+0000 7fe2cb26d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.656+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e8195ef00 seq 10 2019-12-07T19:33:58.656+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.656+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:58.656+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195ef00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195ef00 onodes shared_blobs 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195ef00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.656+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 prepare 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195ef00 2019-12-07T19:33:58.656+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 io_done 2019-12-07T19:33:58.656+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e81942f20 finish 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=22, to_read={1:9b8adabe:::smithi13140796-6:head=0,4096,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=22, to_read={1:9b8adabe:::smithi13140796-6:head=0,4096,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e8192b000 2019-12-07T19:33:58.656+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e81942dc0 prio 127 cost 0 latency 0.000660 MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=22, to_read={1:9b8adabe:::smithi13140796-6:head=0,4096,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.656+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=22, to_read={1:9b8adabe:::smithi13140796-6:head=0,4096,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.656+0000 7fe2cb26d700 25 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.656+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x0~1000 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x6000 (24576) 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.656+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.656+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.656+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.656+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.656+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:33:58.656+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2228 < min_flush_size 524288 2019-12-07T19:33:58.656+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:33:58.656+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2251 < min_flush_size 524288 2019-12-07T19:33:58.656+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2251 < min_flush_size 524288 2019-12-07T19:33:58.656+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x26044~8cb to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x26044 0x26044~8cb to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.656+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x6044 2019-12-07T19:33:58.656+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x44 2019-12-07T19:33:58.656+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.656+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x90f and padding block with 0x6f1 2019-12-07T19:33:58.656+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2690f 2019-12-07T19:33:58.656+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.656+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.656+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.656+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.656+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000118208s (0.000000138s flush + 0.000118070s kv commit) 2019-12-07T19:33:58.656+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1200,0x558e8195ef00> 2019-12-07T19:33:58.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.656+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:33:58.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:33:58.656+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:33:58.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 2019-12-07T19:33:58.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:33:58.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:33:58.656+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:33:58.656+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.656+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 kv_submitted 2019-12-07T19:33:58.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195ef00 2019-12-07T19:33:58.656+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 finishing 2019-12-07T19:33:58.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195ef00 onodes 2019-12-07T19:33:58.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195ef00 done 2019-12-07T19:33:58.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:33:58.656+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195ef00 [] 2019-12-07T19:33:58.656+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.656+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=19, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e811f6800 con 0x558e818d7c00 2019-12-07T19:33:58.657+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.657+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.657+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.657+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.657+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 15 ==== MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=6, reqid=client.4540.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83ac9800 con 0x558e818d7000 2019-12-07T19:33:58.657+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e819438c0 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=6, reqid=client.4540.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.657+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=6, reqid=client.4540.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.658+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e818667f8 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e811713b0 space 0x558e818667f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x0~1000 = 4096 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] get_hash_info: Getting attr on 1:9b8adabe:::smithi13140796-6:head 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] get_hash_info: not in cache 1:9b8adabe:::smithi13140796-6:head 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] get_hash_info: found on disk, size 24576 2019-12-07T19:33:58.659+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# hinfo_key 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# hinfo_key = 0 2019-12-07T19:33:58.659+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 21/18 ECSubReadReply(tid=22, attrs_read=0)) v2 -- 0x558e811dc280 con 0x558e818d7c00 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e81942dc0 finish 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=6, reqid=client.4540.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=6, reqid=client.4540.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e819e4000 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e819438c0 prio 127 cost 0 latency 0.001365 MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=6, reqid=client.4540.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.1s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=6, reqid=client.4540.0:40, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 osd.0 pg_epoch: 21 pg[1.1s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:40 2019-12-07T19:33:58.639869+0000 0 ObjectCleanRegions clean_offsets: [229376~18446744073709322239], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.659+0000 7fe2cb26d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e8195e300 seq 6 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.659+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558e8192b000 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e300 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e300 onodes shared_blobs 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e81943e40 prio 127 cost 0 latency 0.002937 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 prepare 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e300 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 io_done 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=25, reqid=client.4540.0:38, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.659+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e819438c0 finish 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'5], crt=21'5) [21'6 (21'4) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:38 2019-12-07T19:33:58.638318+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'4) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:38 2019-12-07T19:33:58.638318+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.659+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e81a0a600 seq 11 2019-12-07T19:33:58.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.659+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 3 keys 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 3 keys = 0 2019-12-07T19:33:58.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:33:58.659+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.659+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.659+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:58.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2690f~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.659+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2690f 0x2690f~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.659+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.659+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x690f 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.659+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x90f 2019-12-07T19:33:58.659+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.659+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd80 and padding block with 0x280 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a600 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a600 onodes 0x558e811ddb80 shared_blobs 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811ddc88) update 3#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9da56077:::smithi13140796-13:head# is 624 (485 bytes onode + 2 bytes spanning blobs + 137 bytes inline extents) 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 prepare 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a600 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 io_done 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e81943e40 finish 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.659+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=18, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558e81955a00 con 0x558e818d7c00 2019-12-07T19:33:58.659+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.659+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 16 ==== MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=7, reqid=client.4540.0:41, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e83aca000 con 0x558e818d7000 2019-12-07T19:33:58.659+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943a20 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=7, reqid=client.4540.0:41, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.659+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=7, reqid=client.4540.0:41, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=7, reqid=client.4540.0:41, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=7, reqid=client.4540.0:41, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e819e4000 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e81943a20 prio 127 cost 0 latency 0.000056 MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=7, reqid=client.4540.0:41, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.1s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=7, reqid=client.4540.0:41, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:41 2019-12-07T19:33:58.639904+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:41 2019-12-07T19:33:58.639904+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.659+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e83c56300 seq 7 2019-12-07T19:33:58.659+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 3 keys 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 3 keys = 0 2019-12-07T19:33:58.659+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.659+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56300 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56300 onodes 0x558e811dd680 shared_blobs 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dd788) update 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:891f0d13:::smithi13140796-14:head# is 627 (485 bytes onode + 2 bytes spanning blobs + 140 bytes inline extents) 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 prepare 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56300 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 io_done 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.659+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e81943a20 finish 2019-12-07T19:33:58.659+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.659+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x26d80 2019-12-07T19:33:58.659+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.659+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.659+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.659+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000697627s (0.000000135s flush + 0.000697492s kv commit) 2019-12-07T19:33:58.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1664 < min_flush_size 524288 2019-12-07T19:33:58.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811ddb80 had 1 2019-12-07T19:33:58.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:33:58.660+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3331 < min_flush_size 524288 2019-12-07T19:33:58.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dd680 had 1 2019-12-07T19:33:58.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:33:58.660+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3354 < min_flush_size 524288 2019-12-07T19:33:58.660+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3354 < min_flush_size 524288 2019-12-07T19:33:58.660+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.660+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x26d80~d1a to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.660+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x26d80 0x26d80~d1a to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.660+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.660+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x6d80 2019-12-07T19:33:58.660+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd80 2019-12-07T19:33:58.660+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.660+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xa9a and padding block with 0x566 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e300> 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.660+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 kv_submitted 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e300 2019-12-07T19:33:58.660+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 finishing 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e300 onodes 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 done 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56300 kv_submitted 2019-12-07T19:33:58.660+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e300 [] 2019-12-07T19:33:58.660+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.660+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x27a9a 2019-12-07T19:33:58.660+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.660+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.660+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.660+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000117372s (0.000000099s flush + 0.000117273s kv commit) 2019-12-07T19:33:58.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0a600,0x558e83c56300> 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.660+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 kv_submitted 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a600 2019-12-07T19:33:58.660+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 finishing 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a600 onodes 0x558e811ddb80 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a600 done 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:33:58.660+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a600 [] 2019-12-07T19:33:58.660+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.660+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 kv_submitted 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56300 2019-12-07T19:33:58.660+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 finishing 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56300 onodes 0x558e811dd680 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56300 done 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:33:58.660+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56300 [] 2019-12-07T19:33:58.660+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.661+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 21/18 ECSubWriteReply(tid=6, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e8118f400 con 0x558e818d7000 2019-12-07T19:33:58.661+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=25, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558e811f6400 con 0x558e818d7c00 2019-12-07T19:33:58.661+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 21/18 ECSubWriteReply(tid=7, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e81955e00 con 0x558e818d7000 2019-12-07T19:33:58.661+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.662+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.662+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 msgr2=0x558e81847180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 95 2019-12-07T19:33:58.662+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 msgr2=0x558e81847180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.662+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=READY pgs=7 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.662+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=READY pgs=7 cs=0 l=0 rx=0 tx=0)._fault initiating reconnect 2019-12-07T19:33:58.663+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 43 ==== MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=26, reqid=client.4540.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e819da000 con 0x558e818d7c00 2019-12-07T19:33:58.663+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81942b00 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=26, reqid=client.4540.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.663+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=26, reqid=client.4540.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.663+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 44 ==== MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=27, reqid=client.4540.0:44, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e819da800 con 0x558e818d7c00 2019-12-07T19:33:58.663+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81943600 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=27, reqid=client.4540.0:44, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.663+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=27, reqid=client.4540.0:44, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=26, reqid=client.4540.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=26, reqid=client.4540.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e8193b000 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81942b00 prio 127 cost 0 latency 0.000112 MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=26, reqid=client.4540.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.as3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=26, reqid=client.4540.0:43, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.as3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:43 2019-12-07T19:33:58.641743+0000 0 ObjectCleanRegions clean_offsets: [245760~18446744073709305855], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.663+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e8195f500 seq 6 2019-12-07T19:33:58.663+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195f500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195f500 onodes shared_blobs 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195f500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 prepare 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195f500 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 io_done 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81942b00 finish 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=27, reqid=client.4540.0:44, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=27, reqid=client.4540.0:44, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e8193b000 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81943600 prio 127 cost 0 latency 0.000175 MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=27, reqid=client.4540.0:44, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.as3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=27, reqid=client.4540.0:44, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:44 2019-12-07T19:33:58.641780+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:44 2019-12-07T19:33:58.641780+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.663+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e83c57200 seq 7 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x27a9a~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.663+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# 3 keys 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x27a9a 0x27a9a~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# 3 keys = 0 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.663+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x7a9a 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xa9a 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xf0b and padding block with 0xf5 2019-12-07T19:33:58.663+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c57200 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c57200 onodes 0x558e83c72a00 shared_blobs 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e83c72b08) update 3#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:51ccf92f:::smithi13140796-15:head# is 643 (485 bytes onode + 2 bytes spanning blobs + 156 bytes inline extents) 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c57200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57200 prepare 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c57200 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57200 io_done 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.663+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81943600 finish 2019-12-07T19:33:58.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.663+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 11 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:52, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 75700+0+0 (unknown 0 0 0) 0x558e83aca800 con 0x558e818fe400 2019-12-07T19:33:58.663+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943b80 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:52, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.663+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:52, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x27f0b 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000120485s (0.000000143s flush + 0.000120342s kv commit) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:52, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 245 < min_flush_size 524288 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1694 < min_flush_size 524288 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:52, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72a00 had 1 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1717 < min_flush_size 524288 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1717 < min_flush_size 524288 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e81943b80 prio 127 cost 0 latency 0.000076 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:52, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 osd.0 21 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 19 -> 21 (as per caller) 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x27f0b~6b5 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x27f0b 0x27f0b~6b5 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:52, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x7f0b 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xf0b 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x5c0 and padding block with 0xa40 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:52 2019-12-07T19:33:58.659063+0000 0 ObjectCleanRegions clean_offsets: [294912~18446744073709256703], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:52 2019-12-07T19:33:58.659063+0000 0 ObjectCleanRegions clean_offsets: [294912~18446744073709256703], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195f500> 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.663+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 kv_submitted 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195f500 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.663+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 finishing 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195f500 onodes 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f500 done 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57200 kv_submitted 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.663+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195f500 [] 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.663+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.663+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e83c57b00 seq 5 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs3_head 0x558e819c41e0) get_onode oid 3#1:db85c185:::smithi13140796-18:head# key 0x838000000000000001db85c18521736d'ithi13140796-18!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.663+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1088 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# = 0 2019-12-07T19:33:58.663+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 2 keys 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 2 keys = 0 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x285c0 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.663+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.663+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.663+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 0x0~12000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:db85c185:::smithi13140796-18:head# 0x0~12000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000154589s (0.000000059s flush + 0.000154530s kv commit) 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~12000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c57200> 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.663+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57200 kv_submitted 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c57200 2019-12-07T19:33:58.663+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57200 finishing 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c57200 onodes 0x558e83c72a00 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57200 done 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:33:58.663+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c57200 [] 2019-12-07T19:33:58.663+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83c57b00 9 blobs 2019-12-07T19:33:58.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate 0x12000/1000,12000,0 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x79000~12000/1000,12000,0 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x79000~12000] 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac4cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac4cb0 blob([0x79000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4c40 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac4cb0) get_ref 0x0~2000 Blob(0x558e83ac4cb0 blob([0x79000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4c40 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac4cb0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83ac4cb0 blob([0x79000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4c40 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ac4c58 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac4850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194f650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac4850 blob([0x7b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f650 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac4850) get_ref 0x0~2000 Blob(0x558e83ac4850 blob([0x7b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f650 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac4850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83ac4850 blob([0x7b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f650 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e8194f668 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194f5e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194f5e0 blob([0x7d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4d90 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e8194f5e0) get_ref 0x0~2000 Blob(0x558e8194f5e0 blob([0x7d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4d90 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e8194f5e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e8194f5e0 blob([0x7d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4d90 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ac4da8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac4e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac4e00 blob([0x7f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4e70 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac4e00) get_ref 0x0~2000 Blob(0x558e83ac4e00 blob([0x7f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4e70 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac4e00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83ac4e00 blob([0x7f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4e70 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ac4e88 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac4ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac4ee0 blob([0x81000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4f50 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac4ee0) get_ref 0x0~2000 Blob(0x558e83ac4ee0 blob([0x81000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4f50 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac4ee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83ac4ee0 blob([0x81000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4f50 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ac4f68 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac4fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac4fc0 blob([0x83000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5030 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac4fc0) get_ref 0x0~2000 Blob(0x558e83ac4fc0 blob([0x83000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5030 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac4fc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83ac4fc0 blob([0x83000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5030 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ac5048 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac50a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac50a0 blob([0x85000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5110 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac50a0) get_ref 0x0~2000 Blob(0x558e83ac50a0 blob([0x85000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5110 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac50a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83ac50a0 blob([0x85000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5110 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ac5128 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac5180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac51f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac5180 blob([0x87000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac51f0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac5180) get_ref 0x0~2000 Blob(0x558e83ac5180 blob([0x87000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac51f0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac5180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83ac5180 blob([0x87000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac51f0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ac5208 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac5260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac52d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac5260 blob([0x89000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac52d0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac5260) get_ref 0x0~2000 Blob(0x558e83ac5260 blob([0x89000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac52d0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.blob(0x558e83ac5260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83ac5260 blob([0x89000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac52d0 sbid 0x0)) 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ac52e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x12000 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83c73508) dirty_range mark inline shard dirty 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 0x0~12000 = 0 2019-12-07T19:33:58.663+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.663+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.663+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.663+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:33:58.664+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.664+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.664+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.664+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.664+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c57b00 cost 116158 (10 ios * 4000 + 76158 bytes) 2019-12-07T19:33:58.664+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c57b00 onodes 0x558e83c73400 shared_blobs 2019-12-07T19:33:58.664+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83c73508) update 3#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:33:58.664+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83c73508) update inline shard 174 bytes from 9 extents 2019-12-07T19:33:58.664+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:db85c185:::smithi13140796-18:head# is 593 (413 bytes onode + 2 bytes spanning blobs + 178 bytes inline extents) 2019-12-07T19:33:58.664+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c57b00 allocated 0x[79000~12000] released 0x[] 2019-12-07T19:33:58.664+0000 7fe2c6263700 10 freelist allocate 0x79000~12000 2019-12-07T19:33:58.664+0000 7fe2c6263700 20 freelist _xor first_key 0x0 last_key 0x80000 2019-12-07T19:33:58.664+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57b00 prepare 2019-12-07T19:33:58.664+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83c57b00 2019-12-07T19:33:58.664+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.664+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.664+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e81943b80 finish 2019-12-07T19:33:58.664+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 21/18 ECSubWriteReply(tid=26, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e81965200 con 0x558e818d7c00 2019-12-07T19:33:58.664+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.664+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 21/18 ECSubWriteReply(tid=27, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e81965400 con 0x558e818d7c00 2019-12-07T19:33:58.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.664+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e83ab5400 0x558e81988680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.664+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e83ab5400 0x558e81988680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.664+0000 7fe2e6d3a700 10 osd.0 21 ms_handle_authentication new session 0x558e818c24e0 con 0x558e83ab5400 entity osd.3 addr 2019-12-07T19:33:58.664+0000 7fe2e6d3a700 10 osd.0 21 ms_handle_authentication session 0x558e818c24e0 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.664+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e83ab5400 0x558e81988680 crc :-1 s=SESSION_ACCEPTING pgs=17 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect race detected, replacing existing=0x558e818d7000 socket by this connection's socket 2019-12-07T19:33:58.664+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e83ab5400 0x558e81988680 crc :-1 s=SESSION_ACCEPTING pgs=17 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x558e818d7000 2019-12-07T19:33:58.664+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e83ab5400 0x558e81988680 crc :-1 s=SESSION_ACCEPTING pgs=17 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:58.664+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=READY pgs=17 cs=1 l=0 rx=0 tx=0).ready entity=osd.3 client_cookie=f8db806cc22eb7e9 server_cookie=47e95957322ca882 in_seq=16 out_seq=14 2019-12-07T19:33:58.665+0000 7fe2d4a80700 2 osd.0 21 ms_handle_reset con 0x558e83ab5400 session 0x558e818c24e0 2019-12-07T19:33:58.665+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 17 ==== MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:41, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83acb000 con 0x558e818d7000 2019-12-07T19:33:58.665+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943ce0 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:41, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.665+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 10 ==== osd_map(20..21 src has 1..21) v4 ==== 1230+0+0 (unknown 0 0 0) 0x558e817df680 con 0x558e818fe400 2019-12-07T19:33:58.665+0000 7fe2d4a80700 20 osd.0 21 OSD::ms_dispatch: osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.665+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:41, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.665+0000 7fe2d4a80700 10 osd.0 21 do_waiters -- start 2019-12-07T19:33:58.665+0000 7fe2d4a80700 10 osd.0 21 do_waiters -- finish 2019-12-07T19:33:58.665+0000 7fe2d4a80700 20 osd.0 21 _dispatch 0x558e817df680 osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.665+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57b00 aio_wait 2019-12-07T19:33:58.665+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c57b00 2019-12-07T19:33:58.665+0000 7fe2d4a80700 3 osd.0 21 handle_osd_map epochs [20,21], i have 21, src has [1,21] 2019-12-07T19:33:58.665+0000 7fe2d4a80700 10 osd.0 21 no new maps here, dropping 2019-12-07T19:33:58.665+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57b00 io_done 2019-12-07T19:33:58.665+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 18 ==== MOSDECSubOpRead(1.1s1 21/18 ECSubRead(tid=12, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8191bb80 con 0x558e818d7000 2019-12-07T19:33:58.665+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e819431e0 prio 127 cost 0 latency 0.000015 epoch 21 MOSDECSubOpRead(1.1s1 21/18 ECSubRead(tid=12, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.665+0000 7fe2e6d3a700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 21/18 ECSubRead(tid=12, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:41, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:41, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558e819e4000 2019-12-07T19:33:58.665+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e81943ce0 prio 127 cost 0 latency 0.000090 MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:41, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.665+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:41, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.665+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:41 2019-12-07T19:33:58.639904+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.665+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.665+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.665+0000 7fe2cb26d700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.665+0000 7fe2cb26d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.665+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e8195e600 seq 8 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.665+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.665+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:33:58.665+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e600 onodes shared_blobs 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.665+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 prepare 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e600 2019-12-07T19:33:58.665+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 io_done 2019-12-07T19:33:58.665+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e81943ce0 finish 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 21/18 ECSubRead(tid=12, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 21/18 ECSubRead(tid=12, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e819e4000 2019-12-07T19:33:58.665+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e819431e0 prio 127 cost 0 latency 0.000192 MOSDECSubOpRead(1.1s1 21/18 ECSubRead(tid=12, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.665+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpRead(1.1s1 21/18 ECSubRead(tid=12, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.665+0000 7fe2cb26d700 25 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.665+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x0~1000 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0xe000 (57344) 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65730 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c65730 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.665+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.665+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 45 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 79796+0+0 (unknown 0 0 0) 0x558e819d8800 con 0x558e818d7c00 2019-12-07T19:33:58.665+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81942dc0 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.665+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.665+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81942dc0 prio 127 cost 0 latency 0.000081 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.665+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=30, reqid=client.4540.0:55, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.665+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (0'0) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:55 2019-12-07T19:33:58.661779+0000 0 ObjectCleanRegions clean_offsets: [311296~18446744073709240319], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.665+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:55 2019-12-07T19:33:58.661779+0000 0 ObjectCleanRegions clean_offsets: [311296~18446744073709240319], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2148 < min_flush_size 524288 2019-12-07T19:33:58.665+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.665+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.665+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.665+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c73400 had 1 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2171 < min_flush_size 524288 2019-12-07T19:33:58.665+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2171 < min_flush_size 524288 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e81a0a300 seq 9 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x285c0~87b to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.7s3_head 0x558e8122c5a0) get_onode oid 3#1:ec1285ab:::smithi13140796-19:head# key 0x838000000000000001ec1285ab21736d'ithi13140796-19!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x285c0 0x285c0~87b to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x85c0 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x5c0 2019-12-07T19:33:58.665+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1089 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xe3b and padding block with 0x1c5 2019-12-07T19:33:58.665+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# = 0 2019-12-07T19:33:58.665+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 2 keys 2019-12-07T19:33:58.665+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 2 keys = 0 2019-12-07T19:33:58.665+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.665+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.665+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 0x0~13000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:ec1285ab:::smithi13140796-19:head# 0x0~13000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.665+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~13000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e81a0a300 10 blobs 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.665+0000 7fe2c425f700 10 fbmap_alloc 0x558e80233300 allocate 0x13000/1000,13000,0 2019-12-07T19:33:58.665+0000 7fe2c425f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x8b000~13000/1000,13000,0 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x8b000~13000] 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e8e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af8150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e8e70 blob([0x8b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8150 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e817e8e70) get_ref 0x0~2000 Blob(0x558e817e8e70 blob([0x8b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8150 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e817e8e70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e817e8e70 blob([0x8b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8150 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8168 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af80e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af8070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af80e0 blob([0x8d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8070 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e83af80e0) get_ref 0x0~2000 Blob(0x558e83af80e0 blob([0x8d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8070 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e83af80e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83af80e0 blob([0x8d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8070 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8088 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x28e3b 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac5ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e8f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac5ab0 blob([0x8f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8f50 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e83ac5ab0) get_ref 0x0~2000 Blob(0x558e83ac5ab0 blob([0x8f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8f50 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e83ac5ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83ac5ab0 blob([0x8f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8f50 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e817e8f68 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000682952s (0.000462309s flush + 0.000220643s kv commit) 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e90a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af81c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e90a0 blob([0x91000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af81c0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e817e90a0) get_ref 0x0~2000 Blob(0x558e817e90a0 blob([0x91000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af81c0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e817e90a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e817e90a0 blob([0x91000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af81c0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af81d8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af8230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af82a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x28e3b~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af8230 blob([0x93000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af82a0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e83af8230) get_ref 0x0~2000 Blob(0x558e83af8230 blob([0x93000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af82a0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x28e3b 0x28e3b~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e83af8230) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83af8230 blob([0x93000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af82a0 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x8e3b 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xe3b 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af82b8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.665+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x2ac and padding block with 0xd54 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af8310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af8380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af8310 blob([0x95000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8380 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e83af8310) get_ref 0x0~2000 Blob(0x558e83af8310 blob([0x95000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8380 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e83af8310) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83af8310 blob([0x95000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8380 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8398 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af83f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af8460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af83f0 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8460 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e83af83f0) get_ref 0x0~2000 Blob(0x558e83af83f0 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8460 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e83af83f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83af83f0 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8460 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8478 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af84d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af8540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af84d0 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8540 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c57b00> 2019-12-07T19:33:58.665+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.665+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57b00 kv_submitted 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e83af84d0) get_ref 0x0~2000 Blob(0x558e83af84d0 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8540 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e83af84d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83af84d0 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8540 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8558 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c57b00 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af85b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af8620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.665+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57b00 finishing 2019-12-07T19:33:58.665+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c57b00 onodes 0x558e83c73400 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af85b0 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8620 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194f668 in 0x558e80219c00) _finish_write discard buffer(0x558e81171290 space 0x558e8194f668 0x0~2000 writing nocache) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e83af85b0) get_ref 0x0~2000 Blob(0x558e83af85b0 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8620 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac4c58 in 0x558e80219c00) _finish_write discard buffer(0x558e81170900 space 0x558e83ac4c58 0x0~2000 writing nocache) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.blob(0x558e83af85b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.665+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac4da8 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8e480 space 0x558e83ac4da8 0x0~2000 writing nocache) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83af85b0 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8620 sbid 0x0)) 2019-12-07T19:33:58.665+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac4e88 in 0x558e80219c00) _finish_write discard buffer(0x558e81171050 space 0x558e83ac4e88 0x0~2000 writing nocache) 2019-12-07T19:33:58.665+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8638 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.665+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac4f68 in 0x558e80219c00) _finish_write discard buffer(0x558e811715f0 space 0x558e83ac4f68 0x0~2000 writing nocache) 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac5048 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8e630 space 0x558e83ac5048 0x0~2000 writing nocache) 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac5128 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8e6c0 space 0x558e83ac5128 0x0~2000 writing nocache) 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac5208 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8e750 space 0x558e83ac5208 0x0~2000 writing nocache) 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac52e8 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8e7e0 space 0x558e83ac52e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af8690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af8700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57b00 done 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:58.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c57b00 [] 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af8690 blob([0x9d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8700 sbid 0x0)) 2019-12-07T19:33:58.666+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore.blob(0x558e83af8690) get_ref 0x0~1000 Blob(0x558e83af8690 blob([0x9d000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8700 sbid 0x0)) 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore.blob(0x558e83af8690) get_ref init 0x1000, 1000 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~1000: 0x0~1000 Blob(0x558e83af8690 blob([0x9d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83af8700 sbid 0x0)) 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8718 in 0x558e80219880) _discard 0x0~1000 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x13000 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c73c88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.666+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 0x0~13000 = 0 2019-12-07T19:33:58.666+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e81954a00 con 0x558e818fe400 2019-12-07T19:33:58.666+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.666+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.666+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.666+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.666+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.666+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a300 cost 123446 (11 ios * 4000 + 79446 bytes) 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a300 onodes 0x558e83c73b80 shared_blobs 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c73c88) update 3#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c73c88) update inline shard 190 bytes from 10 extents 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:ec1285ab:::smithi13140796-19:head# is 609 (413 bytes onode + 2 bytes spanning blobs + 194 bytes inline extents) 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a300 allocated 0x[8b000~13000] released 0x[] 2019-12-07T19:33:58.666+0000 7fe2c425f700 10 freelist allocate 0x8b000~13000 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.666+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x292ac 2019-12-07T19:33:58.666+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.666+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.666+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.666+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.666+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 prepare 2019-12-07T19:33:58.666+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e81a0a300 2019-12-07T19:33:58.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000199970s (0.000000084s flush + 0.000199886s kv commit) 2019-12-07T19:33:58.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e600> 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 kv_submitted 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e600 2019-12-07T19:33:58.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 finishing 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e600 onodes 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e600 done 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:33:58.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e600 [] 2019-12-07T19:33:58.666+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.666+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.666+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81942dc0 finish 2019-12-07T19:33:58.666+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.666+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.666+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 aio_wait 2019-12-07T19:33:58.666+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a300 2019-12-07T19:33:58.666+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 io_done 2019-12-07T19:33:58.666+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 46 ==== MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e811b8000 con 0x558e818d7c00 2019-12-07T19:33:58.666+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81942f20 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.666+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.666+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 47 ==== MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=32, to_read={1:9da56077:::smithi13140796-13:head=0,4096,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e811dc280 con 0x558e818d7c00 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x558e8192b000 2019-12-07T19:33:58.666+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e818058c0 prio 127 cost 0 latency 0.000015 epoch 21 MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=32, to_read={1:9da56077:::smithi13140796-13:head=0,4096,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.666+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=32, to_read={1:9da56077:::smithi13140796-13:head=0,4096,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.666+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e81942f20 prio 127 cost 0 latency 0.000050 MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.666+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.9s3 21/18 ECSubWrite(tid=31, reqid=client.4540.0:38, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.666+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'6], crt=21'5) [] 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'6 (21'4) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:38 2019-12-07T19:33:58.638318+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.666+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.666+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.666+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.666+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.666+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83c56600 seq 12 2019-12-07T19:33:58.666+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.666+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:33:58.666+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56600 onodes shared_blobs 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.666+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 prepare 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56600 2019-12-07T19:33:58.666+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 io_done 2019-12-07T19:33:58.666+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e81942f20 finish 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=32, to_read={1:9da56077:::smithi13140796-13:head=0,4096,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=32, to_read={1:9da56077:::smithi13140796-13:head=0,4096,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e8192b000 2019-12-07T19:33:58.666+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e818058c0 prio 127 cost 0 latency 0.000166 MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=32, to_read={1:9da56077:::smithi13140796-13:head=0,4096,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.666+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=32, to_read={1:9da56077:::smithi13140796-13:head=0,4096,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.666+0000 7fe2c7265700 25 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.666+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x0~1000 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0xd000 (53248) 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.666+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.667+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1321 < min_flush_size 524288 2019-12-07T19:33:58.667+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c73b80 had 1 2019-12-07T19:33:58.667+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.667+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2435 < min_flush_size 524288 2019-12-07T19:33:58.667+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:33:58.667+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2458 < min_flush_size 524288 2019-12-07T19:33:58.667+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2458 < min_flush_size 524288 2019-12-07T19:33:58.667+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.667+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x292ac~99a to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.667+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x292ac 0x292ac~99a to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.667+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.667+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x92ac 2019-12-07T19:33:58.667+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x2ac 2019-12-07T19:33:58.667+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.667+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc46 and padding block with 0x3ba 2019-12-07T19:33:58.667+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x29c46 2019-12-07T19:33:58.667+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.667+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.667+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.667+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.667+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000467136s (0.000261224s flush + 0.000205912s kv commit) 2019-12-07T19:33:58.667+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0a300,0x558e83c56600> 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 kv_submitted 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a300 2019-12-07T19:33:58.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 finishing 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a300 onodes 0x558e83c73b80 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e8f68 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f0e0 space 0x558e817e8f68 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af8088 in 0x558e80219880) _finish_write discard buffer(0x558e83c8eab0 space 0x558e83af8088 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af8168 in 0x558e80219880) _finish_write discard buffer(0x558e83c8ea20 space 0x558e83af8168 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af81d8 in 0x558e80219880) _finish_write discard buffer(0x558e83c8e510 space 0x558e83af81d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af82b8 in 0x558e80219880) _finish_write discard buffer(0x558e83c8e5a0 space 0x558e83af82b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af8398 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f320 space 0x558e83af8398 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af8478 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f3b0 space 0x558e83af8478 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af8558 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f440 space 0x558e83af8558 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af8638 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f4d0 space 0x558e83af8638 0x0~2000 writing nocache) 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af8718 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f560 space 0x558e83af8718 0x0~1000 writing nocache) 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a300 done 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a300 [] 2019-12-07T19:33:58.667+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 kv_submitted 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56600 2019-12-07T19:33:58.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 finishing 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56600 onodes 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56600 done 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:33:58.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56600 [] 2019-12-07T19:33:58.667+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.667+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=30, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e819dea00 con 0x558e818d7c00 2019-12-07T19:33:58.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.667+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.667+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83c64da8 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:33:58.667+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e83c8ed80 space 0x558e83c64da8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.667+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x0~1000 = 4096 2019-12-07T19:33:58.667+0000 7fe2c7265700 20 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.667+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: Getting attr on 1:9da56077:::smithi13140796-13:head 2019-12-07T19:33:58.667+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: not in cache 1:9da56077:::smithi13140796-13:head 2019-12-07T19:33:58.667+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:33:58.667+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: found on disk, size 53248 2019-12-07T19:33:58.667+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key 2019-12-07T19:33:58.667+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key = 0 2019-12-07T19:33:58.667+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 21/18 ECSubReadReply(tid=32, attrs_read=0)) v2 -- 0x558e819b6780 con 0x558e818d7c00 2019-12-07T19:33:58.667+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e818058c0 finish 2019-12-07T19:33:58.667+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.668+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c65730 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.668+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e83c64088 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:33:58.668+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81171440 space 0x558e83c64088 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.668+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x0~1000 = 4096 2019-12-07T19:33:58.668+0000 7fe2cb26d700 20 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.668+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: Getting attr on 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:33:58.668+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: not in cache 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:33:58.668+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:33:58.668+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: found on disk, size 57344 2019-12-07T19:33:58.668+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key 2019-12-07T19:33:58.668+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key = 0 2019-12-07T19:33:58.668+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.1s0 21/18 ECSubReadReply(tid=12, attrs_read=0)) v2 -- 0x558e81a03680 con 0x558e818d7000 2019-12-07T19:33:58.668+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e819431e0 finish 2019-12-07T19:33:58.668+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 21/18 ECSubWriteReply(tid=11, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e81239600 con 0x558e818d7000 2019-12-07T19:33:58.668+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 21/18 ECSubWriteReply(tid=31, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558e811f7800 con 0x558e818d7c00 2019-12-07T19:33:58.668+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.670+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 48 ==== MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=33, reqid=client.4540.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83aca800 con 0x558e818d7c00 2019-12-07T19:33:58.670+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e819438c0 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=33, reqid=client.4540.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.670+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=33, reqid=client.4540.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.670+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 49 ==== MOSDECSubOpRead(1.as3 21/18 ECSubRead(tid=34, to_read={1:51ccf92f:::smithi13140796-15:head=0,4096,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e83b08000 con 0x558e818d7c00 2019-12-07T19:33:58.670+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81943a20 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpRead(1.as3 21/18 ECSubRead(tid=34, to_read={1:51ccf92f:::smithi13140796-15:head=0,4096,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.670+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpRead(1.as3 21/18 ECSubRead(tid=34, to_read={1:51ccf92f:::smithi13140796-15:head=0,4096,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=33, reqid=client.4540.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=33, reqid=client.4540.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558e8193b000 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e819438c0 prio 127 cost 0 latency 0.000117 MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=33, reqid=client.4540.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.as3 21/18 ECSubWrite(tid=33, reqid=client.4540.0:44, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:51ccf92f:::smithi13140796-15:head by client.4540.0:44 2019-12-07T19:33:58.641780+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.670+0000 7fe2c6a64700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e83c56900 seq 8 2019-12-07T19:33:58.670+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56900 onodes shared_blobs 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 prepare 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56900 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 io_done 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e819438c0 finish 2019-12-07T19:33:58.670+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.670+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpRead(1.as3 21/18 ECSubRead(tid=34, to_read={1:51ccf92f:::smithi13140796-15:head=0,4096,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.670+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpRead(1.as3 21/18 ECSubRead(tid=34, to_read={1:51ccf92f:::smithi13140796-15:head=0,4096,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e8193b000 2019-12-07T19:33:58.670+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943a20 prio 127 cost 0 latency 0.000193 MOSDECSubOpRead(1.as3 21/18 ECSubRead(tid=34, to_read={1:51ccf92f:::smithi13140796-15:head=0,4096,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.670+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.670+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.670+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpRead(1.as3 21/18 ECSubRead(tid=34, to_read={1:51ccf92f:::smithi13140796-15:head=0,4096,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.670+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x29c46~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.670+0000 7fe2c6a64700 25 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.670+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# 0x0~1000 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0xf000 (61440) 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.670+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x29c46 0x29c46~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.670+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.670+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0x9c46 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e92d0 blob([0x6a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64f50 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.670+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc46 2019-12-07T19:33:58.670+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.670+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb7 and padding block with 0xf49 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e92d0 blob([0x6a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64f50 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.670+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2a0b7 2019-12-07T19:33:58.670+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.670+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.670+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.670+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.670+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000171996s (0.000000122s flush + 0.000171874s kv commit) 2019-12-07T19:33:58.670+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e92d0 blob([0x6a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64f50 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83c64f68 in 0x558e80219b20) _discard 0x0~1000 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 _add level 1 near 0 on buffer(0x558e81170990 space 0x558e83c64f68 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# 0x0~1000 = 4096 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: Getting attr on 1:51ccf92f:::smithi13140796-15:head 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: not in cache 1:51ccf92f:::smithi13140796-15:head 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: found on disk, size 61440 2019-12-07T19:33:58.670+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# hinfo_key 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# hinfo_key = 0 2019-12-07T19:33:58.670+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.as0 21/18 ECSubReadReply(tid=34, attrs_read=0)) v2 -- 0x558e83c72280 con 0x558e818d7c00 2019-12-07T19:33:58.670+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943a20 finish 2019-12-07T19:33:58.670+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.671+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 19 ==== MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=14, reqid=client.4540.0:58, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 83892+0+0 (unknown 0 0 0) 0x558e83acb800 con 0x558e818d7000 2019-12-07T19:33:58.671+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e819434a0 prio 127 cost 0 latency 0.000029 epoch 21 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=14, reqid=client.4540.0:58, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.671+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=14, reqid=client.4540.0:58, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56900> 2019-12-07T19:33:58.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.671+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 kv_submitted 2019-12-07T19:33:58.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56900 2019-12-07T19:33:58.671+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 finishing 2019-12-07T19:33:58.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56900 onodes 2019-12-07T19:33:58.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56900 done 2019-12-07T19:33:58.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:33:58.671+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56900 [] 2019-12-07T19:33:58.671+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=14, reqid=client.4540.0:58, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=14, reqid=client.4540.0:58, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e819434a0 prio 127 cost 0 latency 0.000285 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=14, reqid=client.4540.0:58, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=14, reqid=client.4540.0:58, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.671+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.as0 21/18 ECSubWriteReply(tid=33, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e81955a00 con 0x558e818d7c00 2019-12-07T19:33:58.671+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:58 2019-12-07T19:33:58.665940+0000 0 ObjectCleanRegions clean_offsets: [327680~18446744073709223935], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:58 2019-12-07T19:33:58.665940+0000 0 ObjectCleanRegions clean_offsets: [327680~18446744073709223935], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.671+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.671+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e811fbb00 seq 5 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) get_onode oid 1#1:f7919c75:::smithi13140796-20:head# key 0x818000000000000001f7919c'u!smithi13140796-20!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.671+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1090 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# = 0 2019-12-07T19:33:58.671+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 2 keys 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 2 keys = 0 2019-12-07T19:33:58.671+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.671+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x0~14000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 1#1:f7919c75:::smithi13140796-20:head# 0x0~14000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~14000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fbb00 10 blobs 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 fbmap_alloc 0x558e80233300 allocate 0x14000/1000,14000,0 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x9e000~14000/1000,14000,0 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x9e000~14000] 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac49a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194e4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac49a0 blob([0x9e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e4d0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e83ac49a0) get_ref 0x0~2000 Blob(0x558e83ac49a0 blob([0x9e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e4d0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e83ac49a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83ac49a0 blob([0x9e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e8194e4e8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81866af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194f810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81866af0 blob([0xa0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f810 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e81866af0) get_ref 0x0~2000 Blob(0x558e81866af0 blob([0xa0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f810 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e81866af0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e81866af0 blob([0xa0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e8194f828 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194f570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194e8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194f570 blob([0xa2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e8c0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e8194f570) get_ref 0x0~2000 Blob(0x558e8194f570 blob([0xa2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e8c0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e8194f570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e8194f570 blob([0xa2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e8194e8d8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c65b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c65b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c65b90 blob([0xa4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65b20 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e83c65b90) get_ref 0x0~2000 Blob(0x558e83c65b90 blob([0xa4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65b20 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e83c65b90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83c65b90 blob([0xa4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83c65b38 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c65a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c65ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c65a40 blob([0xa6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65ab0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e83c65a40) get_ref 0x0~2000 Blob(0x558e83c65a40 blob([0xa6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65ab0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e83c65a40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83c65a40 blob([0xa6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83c65ac8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c64d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c65c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c64d20 blob([0xa8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65c00 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e83c64d20) get_ref 0x0~2000 Blob(0x558e83c64d20 blob([0xa8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65c00 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e83c64d20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83c64d20 blob([0xa8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83c65c18 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e8d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af92d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e8d90 blob([0xaa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af92d0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e817e8d90) get_ref 0x0~2000 Blob(0x558e817e8d90 blob([0xaa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af92d0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e817e8d90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e817e8d90 blob([0xaa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af92e8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af9340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af93b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af9340 blob([0xac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af93b0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9340) get_ref 0x0~2000 Blob(0x558e83af9340 blob([0xac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af93b0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83af9340 blob([0xac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af93c8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af9420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af9490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af9420 blob([0xae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9490 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9420) get_ref 0x0~2000 Blob(0x558e83af9420 blob([0xae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9490 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83af9420 blob([0xae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af94a8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af9500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af9570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af9500 blob([0xb0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9570 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9500) get_ref 0x0~2000 Blob(0x558e83af9500 blob([0xb0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9570 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83af9500 blob([0xb0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 sbid 0x0)) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af9588 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x14000 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83c72608) dirty_range mark inline shard dirty 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x0~14000 = 0 2019-12-07T19:33:58.671+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.671+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:58.671+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fbb00 cost 128350 (11 ios * 4000 + 84350 bytes) 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fbb00 onodes 0x558e83c72500 shared_blobs 2019-12-07T19:33:58.671+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83c72608) update 1#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:33:58.672+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83c72608) update inline shard 193 bytes from 10 extents 2019-12-07T19:33:58.672+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:f7919c75:::smithi13140796-20:head# is 612 (413 bytes onode + 2 bytes spanning blobs + 197 bytes inline extents) 2019-12-07T19:33:58.672+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fbb00 allocated 0x[9e000~14000] released 0x[] 2019-12-07T19:33:58.672+0000 7fe2c8267700 10 freelist allocate 0x9e000~14000 2019-12-07T19:33:58.672+0000 7fe2c8267700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.672+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 prepare 2019-12-07T19:33:58.672+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e811fbb00 2019-12-07T19:33:58.672+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.672+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e819434a0 finish 2019-12-07T19:33:58.672+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.672+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 aio_wait 2019-12-07T19:33:58.672+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fbb00 2019-12-07T19:33:58.672+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 io_done 2019-12-07T19:33:58.672+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.672+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.672+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.672+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2138 < min_flush_size 524288 2019-12-07T19:33:58.672+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72500 had 1 2019-12-07T19:33:58.672+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:58.672+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2161 < min_flush_size 524288 2019-12-07T19:33:58.672+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2161 < min_flush_size 524288 2019-12-07T19:33:58.672+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.672+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2a0b7~871 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.672+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2a0b7 0x2a0b7~871 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.672+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.672+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xa0b7 2019-12-07T19:33:58.672+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb7 2019-12-07T19:33:58.672+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.672+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x928 and padding block with 0x6d8 2019-12-07T19:33:58.672+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2a928 2019-12-07T19:33:58.672+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.672+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.672+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.672+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.672+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000124644s (0.000034108s flush + 0.000090536s kv commit) 2019-12-07T19:33:58.672+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.673+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 50 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83ac9000 con 0x558e818d7c00 2019-12-07T19:33:58.673+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81943080 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.673+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.673+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 51 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e819db800 con 0x558e818d7c00 2019-12-07T19:33:58.673+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e819431e0 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.673+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81943080 prio 127 cost 0 latency 0.000121 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.7s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=36, reqid=client.4540.0:55, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [] 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:55 2019-12-07T19:33:58.661779+0000 0 ObjectCleanRegions clean_offsets: [311296~18446744073709240319], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.673+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e81a0b800 seq 10 2019-12-07T19:33:58.673+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b800 onodes shared_blobs 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 prepare 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b800 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 io_done 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81943080 finish 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e819431e0 prio 127 cost 0 latency 0.000226 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.7s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=37, reqid=client.4540.0:56, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'3], crt=21'3) [21'4 (21'3) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:56 2019-12-07T19:33:58.661803+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] add_log_entry 21'4 (21'3) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:56 2019-12-07T19:33:58.661803+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.673+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e83c56300 seq 11 2019-12-07T19:33:58.673+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 3 keys 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 3 keys = 0 2019-12-07T19:33:58.673+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.673+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56300 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56300 onodes 0x558e83c73b80 shared_blobs 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c73c88) update 3#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:ec1285ab:::smithi13140796-19:head# is 681 (485 bytes onode + 2 bytes spanning blobs + 194 bytes inline extents) 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 prepare 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56300 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 io_done 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.673+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e819431e0 finish 2019-12-07T19:33:58.673+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.673+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.673+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.673+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 2 2019-12-07T19:33:58.673+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2835 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c73b80 had 1 2019-12-07T19:33:58.673+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.673+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2858 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2858 < min_flush_size 524288 2019-12-07T19:33:58.673+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.673+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2a928~b2a to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.673+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2a928 0x2a928~b2a to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.673+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.673+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xa928 2019-12-07T19:33:58.673+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x928 2019-12-07T19:33:58.673+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.673+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x452 and padding block with 0xbae 2019-12-07T19:33:58.673+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2b452 2019-12-07T19:33:58.673+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.673+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.673+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.673+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.673+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000110931s (0.000000171s flush + 0.000110760s kv commit) 2019-12-07T19:33:58.673+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.674+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 6 ==== osd_map(20..21 src has 1..21) v4 ==== 1230+0+0 (unknown 0 0 0) 0x558e8123afc0 con 0x558e818fe000 2019-12-07T19:33:58.674+0000 7fe2d4a80700 20 osd.0 21 OSD::ms_dispatch: osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.674+0000 7fe2d4a80700 10 osd.0 21 do_waiters -- start 2019-12-07T19:33:58.674+0000 7fe2d4a80700 10 osd.0 21 do_waiters -- finish 2019-12-07T19:33:58.674+0000 7fe2d4a80700 20 osd.0 21 _dispatch 0x558e8123afc0 osd_map(20..21 src has 1..21) v4 2019-12-07T19:33:58.674+0000 7fe2d4a80700 3 osd.0 21 handle_osd_map epochs [20,21], i have 21, src has [1,21] 2019-12-07T19:33:58.674+0000 7fe2d4a80700 10 osd.0 21 no new maps here, dropping 2019-12-07T19:33:58.674+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 12 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:52, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83ac9800 con 0x558e818fe400 2019-12-07T19:33:58.674+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81805ce0 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:52, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.674+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:52, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.674+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 7 ==== MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:64, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 92084+0+0 (unknown 0 0 0) 0x558e819db000 con 0x558e818fe000 2019-12-07T19:33:58.674+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0a000 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:64, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.674+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:64, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.674+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.674+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:52, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.674+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.674+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:52, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:64, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:64, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558e81938000 2019-12-07T19:33:58.674+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e81805ce0 prio 127 cost 0 latency 0.000072 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:52, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.bs3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.674+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:52, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 osd.0 21 dequeue_op 0x558e83b0a000 prio 127 cost 0 latency 0.000058 MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:64, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.674+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 osd.0 21 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 19 -> 21 (as per caller) 2019-12-07T19:33:58.674+0000 7fe2ca26b700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:52 2019-12-07T19:33:58.659063+0000 0 ObjectCleanRegions clean_offsets: [294912~18446744073709256703], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 21 pg[1.4s1( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:64, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.674+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.674+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.674+0000 7fe2ca26b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.674+0000 7fe2ca26b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 21 pg[1.4s1( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:64 2019-12-07T19:33:58.671144+0000 0 ObjectCleanRegions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:64 2019-12-07T19:33:58.671144+0000 0 ObjectCleanRegions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.674+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.674+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e8195e900 seq 6 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.674+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.674+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.674+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.674+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.674+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.674+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.674+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e900 onodes shared_blobs 2019-12-07T19:33:58.674+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.674+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 prepare 2019-12-07T19:33:58.674+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e900 2019-12-07T19:33:58.674+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 io_done 2019-12-07T19:33:58.674+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e81805ce0 finish 2019-12-07T19:33:58.674+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e811fac00 seq 5 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) get_onode oid 1#1:276d5164:::smithi13140796-22:head# key 0x818000000000000001276d51'd!smithi13140796-22!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1091 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# = 0 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 2 keys 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 2 keys = 0 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 0x0~16000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 1#1:276d5164:::smithi13140796-22:head# 0x0~16000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~16000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fac00 11 blobs 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 fbmap_alloc 0x558e80233300 allocate 0x16000/1000,16000,0 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 fbmap_alloc 0x558e80233300 allocate extent: 0xb2000~16000/1000,16000,0 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0xb2000~16000] 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e9f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af8000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e9f10 blob([0xb2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8000 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e817e9f10) get_ref 0x0~2000 Blob(0x558e817e9f10 blob([0xb2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8000 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e817e9f10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e817e9f10 blob([0xb2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8000 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e83af8018 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e9ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194efc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e9ea0 blob([0xb4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194efc0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e817e9ea0) get_ref 0x0~2000 Blob(0x558e817e9ea0 blob([0xb4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194efc0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e817e9ea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e817e9ea0 blob([0xb4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194efc0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e8194efd8 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194f110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194f1f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194f110 blob([0xb6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f1f0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8194f110) get_ref 0x0~2000 Blob(0x558e8194f110 blob([0xb6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f1f0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8194f110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e8194f110 blob([0xb6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f1f0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e8194f208 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d36000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d36070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d36000 blob([0xb8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36070 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d36000) get_ref 0x0~2000 Blob(0x558e83d36000 blob([0xb8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36070 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d36000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83d36000 blob([0xb8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36070 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e83d36088 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d360e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d36150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d360e0 blob([0xba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36150 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d360e0) get_ref 0x0~2000 Blob(0x558e83d360e0 blob([0xba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36150 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d360e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83d360e0 blob([0xba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36150 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e83d36168 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d361c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d36230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d361c0 blob([0xbc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36230 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d361c0) get_ref 0x0~2000 Blob(0x558e83d361c0 blob([0xbc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36230 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d361c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83d361c0 blob([0xbc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36230 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e83d36248 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d362a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d36310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d362a0 blob([0xbe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36310 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d362a0) get_ref 0x0~2000 Blob(0x558e83d362a0 blob([0xbe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36310 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d362a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83d362a0 blob([0xbe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36310 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e83d36328 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d36380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d363f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d36380 blob([0xc0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d363f0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d36380) get_ref 0x0~2000 Blob(0x558e83d36380 blob([0xc0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d363f0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d36380) get_ref init 0x2000, 1000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83d36380 blob([0xc0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d363f0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e83d36408 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d36460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d364d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d36460 blob([0xc2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d364d0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d36460) get_ref 0x0~2000 Blob(0x558e83d36460 blob([0xc2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d364d0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d36460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83d36460 blob([0xc2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d364d0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e83d364e8 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d36540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d365b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d36540 blob([0xc4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d365b0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d36540) get_ref 0x0~2000 Blob(0x558e83d36540 blob([0xc4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d365b0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d36540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83d36540 blob([0xc4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d365b0 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e83d365c8 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d36620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d36690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d36620 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36690 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d36620) get_ref 0x0~2000 Blob(0x558e83d36620 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36690 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d36620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83d36620 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36690 sbid 0x0)) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e83d366a8 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x16000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 0x0~16000 = 0 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fac00 cost 140542 (12 ios * 4000 + 92542 bytes) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fac00 onodes 0x558e83b08c80 shared_blobs 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) update 1#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) update inline shard 212 bytes from 11 extents 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:276d5164:::smithi13140796-22:head# is 631 (413 bytes onode + 2 bytes spanning blobs + 216 bytes inline extents) 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fac00 allocated 0x[b2000~16000] released 0x[] 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 freelist allocate 0xb2000~16000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 prepare 2019-12-07T19:33:58.674+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e811fac00 2019-12-07T19:33:58.675+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.675+0000 7fe2c9a6a700 10 osd.0 21 dequeue_op 0x558e83b0a000 finish 2019-12-07T19:33:58.675+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.675+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 aio_wait 2019-12-07T19:33:58.675+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fac00 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.675+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 io_done 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2b452~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2b452 0x2b452~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xb452 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x452 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x8c3 and padding block with 0x73d 2019-12-07T19:33:58.675+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 13 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:53, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e83ac8800 con 0x558e818fe400 2019-12-07T19:33:58.675+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e818058c0 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:53, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.675+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:53, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:53, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:53, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e818058c0 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:53, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.bs3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:53, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:53 2019-12-07T19:33:58.659098+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2b8c3 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:53 2019-12-07T19:33:58.659098+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.675+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e83b28000 seq 7 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000241816s (0.000000108s flush + 0.000241708s kv commit) 2019-12-07T19:33:58.675+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 3 keys 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 3 keys = 0 2019-12-07T19:33:58.675+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2157 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.675+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b08c80 had 1 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2180 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2180 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83b28000 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83b28000 onodes 0x558e83c73400 shared_blobs 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2b8c3~884 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83c73508) update 3#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2b8c3 0x2b8c3~884 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:db85c185:::smithi13140796-18:head# is 665 (485 bytes onode + 2 bytes spanning blobs + 178 bytes inline extents) 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xb8c3 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x8c3 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x147 and padding block with 0xeb9 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83b28000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83b28000 prepare 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83b28000 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83b28000 io_done 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.675+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e818058c0 finish 2019-12-07T19:33:58.675+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2c147 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000163036s (0.000003783s flush + 0.000159253s kv commit) 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1705 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c73400 had 1 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1728 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1728 < min_flush_size 524288 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2c147~6c0 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2c147 0x2c147~6c0 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xc147 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x147 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x807 and padding block with 0x7f9 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2c807 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.675+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000121870s (0.000000085s flush + 0.000121785s kv commit) 2019-12-07T19:33:58.675+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fbb00,0x558e81a0b800,0x558e83c56300,0x558e8195e900,0x558e811fac00,0x558e83b28000> 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 kv_submitted 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fbb00 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 finishing 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fbb00 onodes 0x558e83c72500 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194e4e8 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8e3f0 space 0x558e8194e4e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194e8d8 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8f290 space 0x558e8194e8d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194f828 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8f200 space 0x558e8194f828 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af92e8 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8f170 space 0x558e83af92e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af93c8 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8e480 space 0x558e83af93c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af94a8 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8e630 space 0x558e83af94a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=14, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e811f6800 con 0x558e818d7000 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af9588 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8e6c0 space 0x558e83af9588 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c65ac8 in 0x558e8102e000) _finish_write discard buffer(0x558e81171050 space 0x558e83c65ac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c65b38 in 0x558e8102e000) _finish_write discard buffer(0x558e81170900 space 0x558e83c65b38 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c65c18 in 0x558e8102e000) _finish_write discard buffer(0x558e811715f0 space 0x558e83c65c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fbb00 done 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fbb00 [] 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 kv_submitted 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b800 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 finishing 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b800 onodes 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 done 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56300 kv_submitted 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b800 [] 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 kv_submitted 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56300 2019-12-07T19:33:58.677+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=36, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e81239600 con 0x558e818d7c00 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 finishing 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56300 onodes 0x558e83c73b80 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56300 done 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56300 [] 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 kv_submitted 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e900 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 finishing 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e900 onodes 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e900 done 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83b28000 kv_submitted 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e900 [] 2019-12-07T19:33:58.677+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=37, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558e818eee00 con 0x558e818d7c00 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 kv_submitted 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fac00 2019-12-07T19:33:58.677+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 finishing 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fac00 onodes 0x558e83b08c80 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194efd8 in 0x558e80219260) _finish_write discard buffer(0x558e83c8f7a0 space 0x558e8194efd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194f208 in 0x558e80219260) _finish_write discard buffer(0x558e83c8f830 space 0x558e8194f208 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af8018 in 0x558e80219260) _finish_write discard buffer(0x558e83c8f710 space 0x558e83af8018 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d36088 in 0x558e80219260) _finish_write discard buffer(0x558e83c8f8c0 space 0x558e83d36088 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d36168 in 0x558e80219260) _finish_write discard buffer(0x558e83c8f950 space 0x558e83d36168 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d36248 in 0x558e80219260) _finish_write discard buffer(0x558e83c8f9e0 space 0x558e83d36248 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d36328 in 0x558e80219260) _finish_write discard buffer(0x558e83c8fa70 space 0x558e83d36328 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d36408 in 0x558e80219260) _finish_write discard buffer(0x558e83c8fb00 space 0x558e83d36408 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d364e8 in 0x558e80219260) _finish_write discard buffer(0x558e83c8fb90 space 0x558e83d364e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d365c8 in 0x558e80219260) _finish_write discard buffer(0x558e83c8fc20 space 0x558e83d365c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d366a8 in 0x558e80219260) _finish_write discard buffer(0x558e83c8fcb0 space 0x558e83d366a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fac00 done 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df900 q now empty 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fac00 [] 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83b28000 kv_submitted 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83b28000 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83b28000 finishing 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83b28000 onodes 0x558e83c73400 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83b28000 done 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83b28000 [] 2019-12-07T19:33:58.677+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.677+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=4, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e80264c00 con 0x558e818fe400 2019-12-07T19:33:58.677+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=5, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e81954c00 con 0x558e818fe400 2019-12-07T19:33:58.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7fe2c9a6a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e81964c00 con 0x558e818fe000 2019-12-07T19:33:58.677+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 20 ==== MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83ac8000 con 0x558e818d7000 2019-12-07T19:33:58.677+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943760 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.677+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.677+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81943760 prio 127 cost 0 latency 0.000085 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.fs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.677+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=16, reqid=client.4540.0:58, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.677+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.fs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:58 2019-12-07T19:33:58.665940+0000 0 ObjectCleanRegions clean_offsets: [327680~18446744073709223935], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.677+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 msgr2=0x558e81847180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 95 2019-12-07T19:33:58.677+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.677+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 msgr2=0x558e81847180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.677+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.677+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.677+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=READY pgs=17 cs=1 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.677+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.677+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e819a1200 seq 6 2019-12-07T19:33:58.677+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:58.677+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=READY pgs=17 cs=1 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.677+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:58.677+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes shared_blobs 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.677+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:33:58.677+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:33:58.677+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81943760 finish 2019-12-07T19:33:58.677+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.677+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 52 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e819d9800 con 0x558e818d7c00 2019-12-07T19:33:58.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.677+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81943ce0 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.677+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:58.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.677+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.677+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2c807~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.677+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2c807 0x2c807~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.677+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.677+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.677+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xc807 2019-12-07T19:33:58.677+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.677+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x807 2019-12-07T19:33:58.677+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.677+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc78 and padding block with 0x388 2019-12-07T19:33:58.677+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.677+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81943ce0 prio 127 cost 0 latency 0.000062 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.677+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=38, reqid=client.4540.0:56, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.677+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'4], crt=21'3) [] 2019-12-07T19:33:58.677+0000 7fe2c8267700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] rollforward: entry=21'4 (21'3) modify 1:ec1285ab:::smithi13140796-19:head by client.4540.0:56 2019-12-07T19:33:58.661803+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.677+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2cc78 2019-12-07T19:33:58.677+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.677+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.677+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.677+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.677+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.677+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.677+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000103428s (0.000000099s flush + 0.000103329s kv commit) 2019-12-07T19:33:58.677+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.678+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.678+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e811fb800 seq 12 2019-12-07T19:33:58.678+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.678+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.678+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.678+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.678+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.678+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fb800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.678+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fb800 onodes shared_blobs 2019-12-07T19:33:58.678+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fb800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.678+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 prepare 2019-12-07T19:33:58.678+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fb800 2019-12-07T19:33:58.678+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 io_done 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1200> 2019-12-07T19:33:58.678+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81943ce0 finish 2019-12-07T19:33:58.678+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.678+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.678+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2cc78~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.678+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:33:58.678+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2cc78 0x2cc78~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xcc78 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc78 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xe9 and padding block with 0xf17 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.678+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.678+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=16, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e81955a00 con 0x558e818d7000 2019-12-07T19:33:58.678+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.678+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=STANDBY pgs=17 cs=2 l=0 rx=0 tx=0).connect 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2d0e9 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000305245s (0.000000091s flush + 0.000305154s kv commit) 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fb800> 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.678+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 kv_submitted 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fb800 2019-12-07T19:33:58.678+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 finishing 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fb800 onodes 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb800 done 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.678+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fb800 [] 2019-12-07T19:33:58.678+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.678+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 53 ==== MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=39, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e83c72280 con 0x558e818d7c00 2019-12-07T19:33:58.678+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0a160 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=39, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.678+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=39, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.678+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 14 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e811b8800 con 0x558e818fe400 2019-12-07T19:33:58.678+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81942b00 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.678+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.678+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.678+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.678+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.678+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.678+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e81942b00 prio 127 cost 0 latency 0.000060 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.678+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:53, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.678+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.678+0000 7fe2c6263700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:53 2019-12-07T19:33:58.659098+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.678+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.678+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.678+0000 7fe2c6263700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.678+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.678+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.678+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e83b28300 seq 8 2019-12-07T19:33:58.678+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.678+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.678+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.678+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.678+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.678+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83b28300 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.678+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83b28300 onodes shared_blobs 2019-12-07T19:33:58.678+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83b28300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.678+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83b28300 prepare 2019-12-07T19:33:58.678+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83b28300 2019-12-07T19:33:58.678+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83b28300 io_done 2019-12-07T19:33:58.678+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e81942b00 finish 2019-12-07T19:33:58.678+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2d0e9~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2d0e9 0x2d0e9~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xd0e9 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xe9 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x55a and padding block with 0xaa6 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2d55a 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.678+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000128128s (0.000000093s flush + 0.000128035s kv commit) 2019-12-07T19:33:58.678+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83b28300> 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.678+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83b28300 kv_submitted 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83b28300 2019-12-07T19:33:58.678+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83b28300 finishing 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83b28300 onodes 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83b28300 done 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:58.678+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83b28300 [] 2019-12-07T19:33:58.678+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.678+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.678+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=8, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e81239800 con 0x558e818fe400 2019-12-07T19:33:58.678+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.678+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.678+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 15 ==== MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=9, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e819b6000 con 0x558e818fe400 2019-12-07T19:33:58.678+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e819438c0 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=9, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.679+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=9, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=9, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=9, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.679+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e819438c0 prio 127 cost 0 latency 0.000073 MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=9, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.679+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=9, to_read={1:db85c185:::smithi13140796-18:head=0,4096,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.679+0000 7fe2ca26b700 25 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.679+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 0x0~1000 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x12000 (73728) 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4cb0 blob([0x79000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4c40 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac4cb0 blob([0x79000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4c40 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac4cb0 blob([0x79000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4c40 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4c58 in 0x558e80219c00) _discard 0x0~1000 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e81170bd0 space 0x558e83ac4c58 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.679+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 0x0~1000 = 4096 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=39, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.679+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: Getting attr on 1:db85c185:::smithi13140796-18:head 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=39, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.679+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: not in cache 1:db85c185:::smithi13140796-18:head 2019-12-07T19:33:58.679+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:33:58.679+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: found on disk, size 73728 2019-12-07T19:33:58.679+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83b0a160 prio 127 cost 0 latency 0.000812 MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=39, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.679+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# hinfo_key 2019-12-07T19:33:58.679+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# hinfo_key = 0 2019-12-07T19:33:58.679+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=39, to_read={1:ec1285ab:::smithi13140796-19:head=0,4096,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.679+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 -- 0x558e83b08500 con 0x558e818fe400 2019-12-07T19:33:58.679+0000 7fe2c425f700 25 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.679+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 0x0~1000 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x13000 (77824) 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.679+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e819438c0 finish 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8e70 blob([0x8b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8150 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.679+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e8e70 blob([0x8b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8150 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.679+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e83d69800 0x558e81988c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.679+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e83d69800 0x558e81988c00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e8e70 blob([0x8b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8150 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8168 in 0x558e80219880) _discard 0x0~1000 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e811717a0 space 0x558e83af8168 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.679+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 0x0~1000 = 4096 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.679+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: Getting attr on 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:33:58.679+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: not in cache 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:33:58.679+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:33:58.679+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: found on disk, size 77824 2019-12-07T19:33:58.679+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# hinfo_key 2019-12-07T19:33:58.679+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# hinfo_key = 0 2019-12-07T19:33:58.679+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 21/18 ECSubReadReply(tid=39, attrs_read=0)) v2 -- 0x558e83c73900 con 0x558e818d7c00 2019-12-07T19:33:58.679+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83b0a160 finish 2019-12-07T19:33:58.679+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.679+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=38, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558e818ee600 con 0x558e818d7c00 2019-12-07T19:33:58.679+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.679+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 8 ==== MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:64, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e811b8800 con 0x558e818fe000 2019-12-07T19:33:58.679+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e81943340 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:64, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.679+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:64, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.679+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.679+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:64, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.680+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:64, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e81938000 2019-12-07T19:33:58.680+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=BANNER_CONNECTING pgs=17 cs=2 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.680+0000 7fe2c5a62700 10 osd.0 21 dequeue_op 0x558e81943340 prio 127 cost 0 latency 0.000070 MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:64, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.4s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.680+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:64, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.680+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.680+0000 7fe2c5a62700 20 osd.0 pg_epoch: 21 pg[1.4s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:64 2019-12-07T19:33:58.671144+0000 0 ObjectCleanRegions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.680+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.680+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.680+0000 7fe2c5a62700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.680+0000 7fe2c5a62700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.680+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:33:58.680+0000 7fe2e6d3a700 10 osd.0 21 ms_handle_authentication new session 0x558e802456c0 con 0x558e83d69800 entity osd.3 addr 2019-12-07T19:33:58.680+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e811faf00 seq 6 2019-12-07T19:33:58.680+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:33:58.680+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.680+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.680+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.680+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:33:58.680+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811faf00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.680+0000 7fe2e6d3a700 10 osd.0 21 ms_handle_authentication session 0x558e802456c0 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.680+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811faf00 onodes shared_blobs 2019-12-07T19:33:58.680+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811faf00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.680+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 prepare 2019-12-07T19:33:58.680+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811faf00 2019-12-07T19:33:58.680+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 io_done 2019-12-07T19:33:58.680+0000 7fe2c5a62700 10 osd.0 21 dequeue_op 0x558e81943340 finish 2019-12-07T19:33:58.680+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.680+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e83d69800 0x558e81988c00 crc :-1 s=SESSION_ACCEPTING pgs=20 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect race detected, replacing existing=0x558e818d7000 socket by this connection's socket 2019-12-07T19:33:58.680+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e83d69800 0x558e81988c00 crc :-1 s=SESSION_ACCEPTING pgs=20 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x558e818d7000 2019-12-07T19:33:58.680+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e83d69800 0x558e81988c00 crc :-1 s=SESSION_ACCEPTING pgs=20 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:58.680+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 9 ==== MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:65, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e83ac9800 con 0x558e818fe000 2019-12-07T19:33:58.680+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e819434a0 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:65, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.680+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:65, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.680+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] reap_dead start 2019-12-07T19:33:58.680+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=READY pgs=20 cs=2 l=0 rx=0 tx=0).ready entity=osd.3 client_cookie=f8db806cc22eb7e9 server_cookie=47e95957322ca882 in_seq=20 out_seq=17 2019-12-07T19:33:58.680+0000 7fe2d4a80700 2 osd.0 21 ms_handle_reset con 0x558e83d69800 session 0x558e802456c0 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:65, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:65, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e81938000 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 osd.0 21 dequeue_op 0x558e819434a0 prio 127 cost 0 latency 0.000125 MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:65, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.4s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:65, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2d55a~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2d55a 0x2d55a~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:65 2019-12-07T19:33:58.671204+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xd55a 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x55a 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x9cb and padding block with 0x635 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:65 2019-12-07T19:33:58.671204+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e8195fb00 seq 7 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 3 keys 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 3 keys = 0 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195fb00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195fb00 onodes 0x558e83b08c80 shared_blobs 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) update 1#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:276d5164:::smithi13140796-22:head# is 703 (485 bytes onode + 2 bytes spanning blobs + 216 bytes inline extents) 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195fb00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2d9cb 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 prepare 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195fb00 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 io_done 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000163742s (0.000000115s flush + 0.000163627s kv commit) 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 10 osd.0 21 dequeue_op 0x558e819434a0 finish 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1743 < min_flush_size 524288 2019-12-07T19:33:58.680+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 21 ==== MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:59, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e811b9000 con 0x558e818d7000 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b08c80 had 1 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1766 < min_flush_size 524288 2019-12-07T19:33:58.680+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81942420 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:59, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1766 < min_flush_size 524288 2019-12-07T19:33:58.680+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:59, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2d9cb~6e6 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2d9cb 0x2d9cb~6e6 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xd9cb 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x9cb 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb1 and padding block with 0xf4f 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:59, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:59, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811faf00> 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.680+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 kv_submitted 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811faf00 2019-12-07T19:33:58.680+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 finishing 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811faf00 onodes 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811faf00 done 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81942420 prio 127 cost 0 latency 0.000071 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:59, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.fs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 kv_submitted 2019-12-07T19:33:58.680+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811faf00 [] 2019-12-07T19:33:58.680+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=17, reqid=client.4540.0:59, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:59 2019-12-07T19:33:58.665983+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:59 2019-12-07T19:33:58.665983+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.680+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.680+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2e0b1 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.680+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=1, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e818a8800 con 0x558e818fe000 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e811fa900 seq 7 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000147478s (0.000000071s flush + 0.000147407s kv commit) 2019-12-07T19:33:58.680+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.680+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 3 keys 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195fb00> 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.680+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 kv_submitted 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195fb00 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 3 keys = 0 2019-12-07T19:33:58.680+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 finishing 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195fb00 onodes 0x558e83b08c80 2019-12-07T19:33:58.680+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 done 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df900 q now empty 2019-12-07T19:33:58.680+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195fb00 [] 2019-12-07T19:33:58.680+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.680+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.680+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa900 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa900 onodes 0x558e83c72500 shared_blobs 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) update 1#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:f7919c75:::smithi13140796-20:head# is 684 (485 bytes onode + 2 bytes spanning blobs + 197 bytes inline extents) 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 prepare 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa900 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 io_done 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.680+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81942420 finish 2019-12-07T19:33:58.680+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.680+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.681+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.681+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.681+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1724 < min_flush_size 524288 2019-12-07T19:33:58.681+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72500 had 1 2019-12-07T19:33:58.681+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:58.681+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1747 < min_flush_size 524288 2019-12-07T19:33:58.681+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1747 < min_flush_size 524288 2019-12-07T19:33:58.681+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.681+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2e0b1~6d3 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.681+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2e0b1 0x2e0b1~6d3 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.681+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.681+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xe0b1 2019-12-07T19:33:58.681+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb1 2019-12-07T19:33:58.681+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.681+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x784 and padding block with 0x87c 2019-12-07T19:33:58.681+0000 7fe2c9a6a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=2, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e818efe00 con 0x558e818fe000 2019-12-07T19:33:58.681+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.681+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2e784 2019-12-07T19:33:58.681+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.681+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.681+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.681+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.681+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000179771s (0.000000107s flush + 0.000179664s kv commit) 2019-12-07T19:33:58.681+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.681+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.681+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa900> 2019-12-07T19:33:58.681+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.681+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 kv_submitted 2019-12-07T19:33:58.681+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa900 2019-12-07T19:33:58.681+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 finishing 2019-12-07T19:33:58.681+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa900 onodes 0x558e83c72500 2019-12-07T19:33:58.681+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa900 done 2019-12-07T19:33:58.681+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:33:58.681+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa900 [] 2019-12-07T19:33:58.681+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.681+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.681+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.681+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=17, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e818efc00 con 0x558e818d7000 2019-12-07T19:33:58.681+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.684+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 22 ==== MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=18, reqid=client.4540.0:59, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e819db800 con 0x558e818d7000 2019-12-07T19:33:58.684+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a160 prio 127 cost 0 latency 0.000036 epoch 21 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=18, reqid=client.4540.0:59, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.684+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=18, reqid=client.4540.0:59, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.684+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 23 ==== MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=19, to_read={1:f7919c75:::smithi13140796-20:head=0,4096,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e83b08500 con 0x558e818d7000 2019-12-07T19:33:58.684+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a580 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=19, to_read={1:f7919c75:::smithi13140796-20:head=0,4096,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.684+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=19, to_read={1:f7919c75:::smithi13140796-20:head=0,4096,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=18, reqid=client.4540.0:59, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=18, reqid=client.4540.0:59, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83b0a160 prio 127 cost 0 latency 0.001837 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=18, reqid=client.4540.0:59, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=18, reqid=client.4540.0:59, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:59 2019-12-07T19:33:58.665983+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.686+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e811fb500 seq 8 2019-12-07T19:33:58.686+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fb500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fb500 onodes shared_blobs 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fb500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 prepare 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fb500 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 io_done 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83b0a160 finish 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=19, to_read={1:f7919c75:::smithi13140796-20:head=0,4096,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=19, to_read={1:f7919c75:::smithi13140796-20:head=0,4096,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83b0a580 prio 127 cost 0 latency 0.001938 MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=19, to_read={1:f7919c75:::smithi13140796-20:head=0,4096,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=19, to_read={1:f7919c75:::smithi13140796-20:head=0,4096,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.686+0000 7fe2c425f700 25 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.686+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x0~1000 2019-12-07T19:33:58.686+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x14000 (81920) 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.686+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac49a0 blob([0x9e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.686+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac49a0 blob([0x9e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.686+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:58.686+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.686+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.686+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.686+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2e784~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.686+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2e784 0x2e784~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.686+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.686+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xe784 2019-12-07T19:33:58.686+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x784 2019-12-07T19:33:58.686+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.686+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xbf5 and padding block with 0x40b 2019-12-07T19:33:58.686+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2ebf5 2019-12-07T19:33:58.686+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.686+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.686+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.686+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.686+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000159658s (0.000000162s flush + 0.000159496s kv commit) 2019-12-07T19:33:58.686+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.686+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.686+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fb500> 2019-12-07T19:33:58.686+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.686+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 kv_submitted 2019-12-07T19:33:58.686+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fb500 2019-12-07T19:33:58.686+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 finishing 2019-12-07T19:33:58.686+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fb500 onodes 2019-12-07T19:33:58.686+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb500 done 2019-12-07T19:33:58.686+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:33:58.686+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fb500 [] 2019-12-07T19:33:58.686+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.686+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac49a0 blob([0x9e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e8194e4e8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e83c8f5f0 space 0x558e8194e4e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x0~1000 = 4096 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: Getting attr on 1:f7919c75:::smithi13140796-20:head 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: not in cache 1:f7919c75:::smithi13140796-20:head 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: found on disk, size 81920 2019-12-07T19:33:58.686+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key = 0 2019-12-07T19:33:58.686+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.fs0 21/18 ECSubReadReply(tid=19, attrs_read=0)) v2 -- 0x558e83c72280 con 0x558e818d7000 2019-12-07T19:33:58.686+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83b0a580 finish 2019-12-07T19:33:58.686+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.686+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=18, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e81954c00 con 0x558e818d7000 2019-12-07T19:33:58.686+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.689+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 24 ==== MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=20, reqid=client.4540.0:76, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 108468+0+0 (unknown 0 0 0) 0x558e819d8800 con 0x558e818d7000 2019-12-07T19:33:58.689+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943080 prio 127 cost 0 latency 0.000035 epoch 21 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=20, reqid=client.4540.0:76, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.689+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=20, reqid=client.4540.0:76, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=20, reqid=client.4540.0:76, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=20, reqid=client.4540.0:76, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81943080 prio 127 cost 0 latency 0.000109 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=20, reqid=client.4540.0:76, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=20, reqid=client.4540.0:76, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (0'0) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:76 2019-12-07T19:33:58.684279+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:76 2019-12-07T19:33:58.684279+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.689+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e811fbb00 seq 9 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) get_onode oid 1#1:f9bf9ba5:::smithi13140796-26:head# key 0x818000000000000001f9bf9ba521736d'ithi13140796-26!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.689+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1092 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# = 0 2019-12-07T19:33:58.689+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 2 keys 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 2 keys = 0 2019-12-07T19:33:58.689+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.689+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x0~1a000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 1#1:f9bf9ba5:::smithi13140796-26:head# 0x0~1a000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1a000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fbb00 13 blobs 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 fbmap_alloc 0x558e80233300 allocate 0x1a000/1000,1a000,0 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0xc8000~1a000/1000,1a000,0 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0xc8000~1a000] 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194f730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194ec40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194ec40 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e8194f730) get_ref 0x0~2000 Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194ec40 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e8194f730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ec40 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e8194ec58 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d36c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5810 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83d36c40) get_ref 0x0~2000 Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5810 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83d36c40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5810 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5828 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac5730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac57a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac57a0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 54 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 104372+0+0 (unknown 0 0 0) 0x558e819d8000 con 0x558e818d7c00 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83ac5730) get_ref 0x0~2000 Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac57a0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83ac5730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac57a0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0a2c0 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac57b8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac5880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac58f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac58f0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83ac5880) get_ref 0x0~2000 Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac58f0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83ac5880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac58f0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5908 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac5960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5ce0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83ac5960) get_ref 0x0~2000 Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5ce0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83ac5960) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5ce0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5cf8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac4770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5a40 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83ac4770) get_ref 0x0~2000 Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5a40 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83ac4770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5a40 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5a58 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194f180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194ed90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194ed90 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e83b0a2c0 prio 127 cost 0 latency 0.000098 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e8194f180) get_ref 0x0~2000 Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194ed90 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e8194f180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ed90 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e8194eda8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=40, reqid=client.4540.0:73, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af8d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e91f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e91f0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83af8d90) get_ref 0x0~2000 Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e91f0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83af8d90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e91f0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'4], crt=21'4) [21'5 (0'0) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:73 2019-12-07T19:33:58.681506+0000 0 ObjectCleanRegions clean_offsets: [409600~18446744073709142015], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e817e9208 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d37490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d37500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d37500 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37490) get_ref 0x0~2000 Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d37500 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] add_log_entry 21'5 (0'0) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:73 2019-12-07T19:33:58.681506+0000 0 ObjectCleanRegions clean_offsets: [409600~18446744073709142015], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37500 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d37518 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d37570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d375e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.689+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d375e0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37570) get_ref 0x0~2000 Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d375e0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d375e0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d375f8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d37650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d376c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d376c0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37650) get_ref 0x0~2000 Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d376c0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37650) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d376c0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d376d8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e8195e900 seq 13 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d37730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d377a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d377a0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37730) get_ref 0x0~2000 Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d377a0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d377a0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.7s3_head 0x558e8122c5a0) get_onode oid 3#1:ee170468:::smithi13140796-25:head# key 0x838000000000000001ee1704'h!smithi13140796-25!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d377b8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d37810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d37880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1093 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d37880 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# = 0 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37810) get_ref 0x0~2000 Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d37880 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 2 keys 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37880 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d37898 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 2 keys = 0 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1a000 2019-12-07T19:33:58.689+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83b09008) dirty_range mark inline shard dirty 2019-12-07T19:33:58.689+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.689+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 0x0~19000 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x0~1a000 = 0 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:ee170468:::smithi13140796-25:head# 0x0~19000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.689+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.689+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~19000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.689+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8195e900 13 blobs 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fbb00 cost 164118 (14 ios * 4000 + 108118 bytes) 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fbb00 onodes 0x558e83b08f00 shared_blobs 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83b09008) update 1#1:f9bf9ba5:::smithi13140796-26:head# 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83b09008) update inline shard 250 bytes from 13 extents 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:f9bf9ba5:::smithi13140796-26:head# is 669 (413 bytes onode + 2 bytes spanning blobs + 254 bytes inline extents) 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fbb00 allocated 0x[c8000~1a000] released 0x[] 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 freelist allocate 0xc8000~1a000 2019-12-07T19:33:58.689+0000 7fe2c425f700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 prepare 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e811fbb00 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.689+0000 7fe2c8267700 10 fbmap_alloc 0x558e80233300 allocate 0x19000/1000,19000,0 2019-12-07T19:33:58.689+0000 7fe2c8267700 10 fbmap_alloc 0x558e80233300 allocate extent: 0xe2000~19000/1000,19000,0 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0xe2000~19000] 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194f7a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac59d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194f7a0 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac59d0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore.blob(0x558e8194f7a0) get_ref 0x0~2000 Blob(0x558e8194f7a0 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac59d0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore.blob(0x558e8194f7a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e8194f7a0 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac59d0 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83ac59e8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac4a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af9a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac4a80 blob([0xe4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9a40 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore.blob(0x558e83ac4a80) get_ref 0x0~2000 Blob(0x558e83ac4a80 blob([0xe4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9a40 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore.blob(0x558e83ac4a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83ac4a80 blob([0xe4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9a40 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af9a58 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af9b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af9c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af9b90 blob([0xe6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9c00 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9b90) get_ref 0x0~2000 Blob(0x558e83af9b90 blob([0xe6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9c00 sbid 0x0)) 2019-12-07T19:33:58.689+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9b90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83af9b90 blob([0xe6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9c00 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af9c18 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af9c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af9ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af9c70 blob([0xe8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9ce0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9c70) get_ref 0x0~2000 Blob(0x558e83af9c70 blob([0xe8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9ce0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9c70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83af9c70 blob([0xe8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9ce0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af9cf8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af9d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af9dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af9d50 blob([0xea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9dc0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9d50) get_ref 0x0~2000 Blob(0x558e83af9d50 blob([0xea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9dc0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83af9d50 blob([0xea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9dc0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af9dd8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af9e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af9ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af9e30 blob([0xec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9ea0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9e30) get_ref 0x0~2000 Blob(0x558e83af9e30 blob([0xec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9ea0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9e30) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83af9e30 blob([0xec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9ea0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af9eb8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af9f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af9f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af9f10 blob([0xee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9f80 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9f10) get_ref 0x0~2000 Blob(0x558e83af9f10 blob([0xee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9f80 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9f10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83af9f10 blob([0xee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9f80 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af9f98 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0e000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0e000 blob([0xf0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e070 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0e000) get_ref 0x0~2000 Blob(0x558e83d0e000 blob([0xf0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e070 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0e000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83d0e000 blob([0xf0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e070 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0e088 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0e0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0e0e0 blob([0xf2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e150 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0e0e0) get_ref 0x0~2000 Blob(0x558e83d0e0e0 blob([0xf2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e150 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0e0e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83d0e0e0 blob([0xf2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e150 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0e168 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0e1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0e1c0 blob([0xf4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e230 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0e1c0) get_ref 0x0~2000 Blob(0x558e83d0e1c0 blob([0xf4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e230 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0e1c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83d0e1c0 blob([0xf4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e230 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0e248 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0e2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0e2a0 blob([0xf6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e310 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0e2a0) get_ref 0x0~2000 Blob(0x558e83d0e2a0 blob([0xf6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e310 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0e2a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83d0e2a0 blob([0xf6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e310 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0e328 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0e380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0e380 blob([0xf8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e3f0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0e380) get_ref 0x0~2000 Blob(0x558e83d0e380 blob([0xf8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e3f0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0e380) get_ref init 0x2000, 1000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83d0e380 blob([0xf8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e3f0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0e408 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0e460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0e460 blob([0xfa000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e4d0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0e460) get_ref 0x0~1000 Blob(0x558e83d0e460 blob([0xfa000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0e4d0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0e460) get_ref init 0x1000, 1000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~1000: 0x0~1000 Blob(0x558e83d0e460 blob([0xfa000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d0e4d0 sbid 0x0)) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0e4e8 in 0x558e80219880) _discard 0x0~1000 2019-12-07T19:33:58.690+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x19000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83b09a08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.690+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 0x0~19000 = 0 2019-12-07T19:33:58.690+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.690+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.690+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.690+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81943080 finish 2019-12-07T19:33:58.690+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.690+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.690+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e900 cost 160022 (14 ios * 4000 + 104022 bytes) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e900 onodes 0x558e83b09900 shared_blobs 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83b09a08) update 3#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83b09a08) update inline shard 247 bytes from 13 extents 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:ee170468:::smithi13140796-25:head# is 666 (413 bytes onode + 2 bytes spanning blobs + 251 bytes inline extents) 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e900 allocated 0x[e2000~19000] released 0x[] 2019-12-07T19:33:58.690+0000 7fe2c8267700 10 freelist allocate 0xe2000~19000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:58.690+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 prepare 2019-12-07T19:33:58.690+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8195e900 2019-12-07T19:33:58.690+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 aio_wait 2019-12-07T19:33:58.690+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fbb00 2019-12-07T19:33:58.690+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 io_done 2019-12-07T19:33:58.690+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.690+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.690+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.690+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1381 < min_flush_size 524288 2019-12-07T19:33:58.690+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b08f00 had 1 2019-12-07T19:33:58.690+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:58.690+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1404 < min_flush_size 524288 2019-12-07T19:33:58.690+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1404 < min_flush_size 524288 2019-12-07T19:33:58.690+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.690+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2ebf5~57c to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.690+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2ebf5 0x2ebf5~57c to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.690+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.690+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xebf5 2019-12-07T19:33:58.690+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xbf5 2019-12-07T19:33:58.690+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.690+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x171 and padding block with 0xe8f 2019-12-07T19:33:58.690+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2f171 2019-12-07T19:33:58.690+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.690+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.690+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.690+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.690+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000409942s (0.000049509s flush + 0.000360433s kv commit) 2019-12-07T19:33:58.690+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fbb00> 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.690+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 kv_submitted 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fbb00 2019-12-07T19:33:58.690+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 finishing 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fbb00 onodes 0x558e83b08f00 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e9208 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8fdd0 space 0x558e817e9208 0x0~2000 writing nocache) 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194ec58 in 0x558e8102e000) _finish_write discard buffer(0x558e81171290 space 0x558e8194ec58 0x0~2000 writing nocache) 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194eda8 in 0x558e8102e000) _finish_write discard buffer(0x558e811715f0 space 0x558e8194eda8 0x0~2000 writing nocache) 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac57b8 in 0x558e8102e000) _finish_write discard buffer(0x558e81170cf0 space 0x558e83ac57b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac5828 in 0x558e8102e000) _finish_write discard buffer(0x558e81171b00 space 0x558e83ac5828 0x0~2000 writing nocache) 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac5908 in 0x558e8102e000) _finish_write discard buffer(0x558e81170c60 space 0x558e83ac5908 0x0~2000 writing nocache) 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac5a58 in 0x558e8102e000) _finish_write discard buffer(0x558e81170900 space 0x558e83ac5a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac5cf8 in 0x558e8102e000) _finish_write discard buffer(0x558e81171050 space 0x558e83ac5cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d37518 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8f170 space 0x558e83d37518 0x0~2000 writing nocache) 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d375f8 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8e480 space 0x558e83d375f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d376d8 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8e630 space 0x558e83d376d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d377b8 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8e6c0 space 0x558e83d377b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d37898 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8f7a0 space 0x558e83d37898 0x0~2000 writing nocache) 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fbb00 done 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:33:58.690+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fbb00 [] 2019-12-07T19:33:58.690+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.690+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.690+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.690+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e83b0a2c0 finish 2019-12-07T19:33:58.690+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=20, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e818ef600 con 0x558e818d7000 2019-12-07T19:33:58.690+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.690+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 aio_wait 2019-12-07T19:33:58.690+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e900 2019-12-07T19:33:58.690+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 io_done 2019-12-07T19:33:58.691+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 127154 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.691+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.691+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.691+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.692+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1378 < min_flush_size 524288 2019-12-07T19:33:58.692+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b09900 had 1 2019-12-07T19:33:58.692+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.692+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1401 < min_flush_size 524288 2019-12-07T19:33:58.692+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1401 < min_flush_size 524288 2019-12-07T19:33:58.692+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.692+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2f171~579 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.692+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2f171 0x2f171~579 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.692+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.692+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xf171 2019-12-07T19:33:58.692+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x171 2019-12-07T19:33:58.692+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.692+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x6ea and padding block with 0x916 2019-12-07T19:33:58.692+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2f6ea 2019-12-07T19:33:58.692+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.692+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.692+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.692+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.692+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.001324508s (0.001134373s flush + 0.000190135s kv commit) 2019-12-07T19:33:58.692+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e900> 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.692+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 kv_submitted 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e900 2019-12-07T19:33:58.692+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 finishing 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e900 onodes 0x558e83b09900 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac59e8 in 0x558e80219880) _finish_write discard buffer(0x558e83c8e7e0 space 0x558e83ac59e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af9a58 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f0e0 space 0x558e83af9a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af9c18 in 0x558e80219880) _finish_write discard buffer(0x558e83c8eab0 space 0x558e83af9c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af9cf8 in 0x558e80219880) _finish_write discard buffer(0x558e83c8ea20 space 0x558e83af9cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af9dd8 in 0x558e80219880) _finish_write discard buffer(0x558e83c8e510 space 0x558e83af9dd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af9eb8 in 0x558e80219880) _finish_write discard buffer(0x558e83c8e5a0 space 0x558e83af9eb8 0x0~2000 writing nocache) 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af9f98 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f320 space 0x558e83af9f98 0x0~2000 writing nocache) 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0e088 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f3b0 space 0x558e83d0e088 0x0~2000 writing nocache) 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0e168 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f440 space 0x558e83d0e168 0x0~2000 writing nocache) 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0e248 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f4d0 space 0x558e83d0e248 0x0~2000 writing nocache) 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0e328 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f560 space 0x558e83d0e328 0x0~2000 writing nocache) 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0e408 in 0x558e80219880) _finish_write discard buffer(0x558e83c8e3f0 space 0x558e83d0e408 0x0~2000 writing nocache) 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0e4e8 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f290 space 0x558e83d0e4e8 0x0~1000 writing nocache) 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e900 done 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.692+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e900 [] 2019-12-07T19:33:58.692+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.692+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.693+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.693+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=40, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558e818efc00 con 0x558e818d7c00 2019-12-07T19:33:58.693+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.694+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 25 ==== MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=21, reqid=client.4540.0:76, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83acb800 con 0x558e818d7000 2019-12-07T19:33:58.694+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e819431e0 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=21, reqid=client.4540.0:76, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.694+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 10 ==== MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:65, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83ac8800 con 0x558e818fe000 2019-12-07T19:33:58.694+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=21, reqid=client.4540.0:76, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.694+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0a6e0 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:65, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.694+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:65, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.694+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 11 ==== MOSDECSubOpRead(1.4s1 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e819b6500 con 0x558e818fe000 2019-12-07T19:33:58.694+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0a9a0 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpRead(1.4s1 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.694+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpRead(1.4s1 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:65, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:65, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558e81938000 2019-12-07T19:33:58.694+0000 7fe2c5a62700 10 osd.0 21 dequeue_op 0x558e83b0a6e0 prio 127 cost 0 latency 0.000098 MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:65, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.694+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.4s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:65, at_version=0'0, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.694+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [] 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:65 2019-12-07T19:33:58.671204+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.694+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.694+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.694+0000 7fe2c5a62700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.694+0000 7fe2c5a62700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.694+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e811fa600 seq 8 2019-12-07T19:33:58.694+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:33:58.694+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.694+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:33:58.694+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa600 onodes shared_blobs 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.694+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=21, reqid=client.4540.0:76, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.694+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 prepare 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa600 2019-12-07T19:33:58.694+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 io_done 2019-12-07T19:33:58.694+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=21, reqid=client.4540.0:76, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.694+0000 7fe2c5a62700 10 osd.0 21 dequeue_op 0x558e83b0a6e0 finish 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e819431e0 prio 127 cost 0 latency 0.000233 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=21, reqid=client.4540.0:76, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.fs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpRead(1.4s1 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpRead(1.4s1 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e81938000 2019-12-07T19:33:58.694+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=21, reqid=client.4540.0:76, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.694+0000 7fe2c5a62700 10 osd.0 21 dequeue_op 0x558e83b0a9a0 prio 127 cost 0 latency 0.000202 MOSDECSubOpRead(1.4s1 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.694+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [] 2019-12-07T19:33:58.694+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpRead(1.4s1 21/18 ECSubRead(tid=4, to_read={1:276d5164:::smithi13140796-22:head=0,4096,0}, subchunks={1:276d5164:::smithi13140796-22:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.694+0000 7fe2c5a62700 25 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.694+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.fs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:76 2019-12-07T19:33:58.684279+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.694+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 0x0~1000 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x16000 (90112) 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.694+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9f10 blob([0xb2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8000 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.694+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.694+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e9f10 blob([0xb2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8000 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.694+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.694+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:58.694+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.694+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e819a1200 seq 10 2019-12-07T19:33:58.694+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.694+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.694+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:58.694+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:58.694+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.694+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.694+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.694+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:58.694+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:33:58.694+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.694+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.694+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.694+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes shared_blobs 2019-12-07T19:33:58.694+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.694+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.694+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2f6ea~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.694+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2f6ea 0x2f6ea~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.694+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:33:58.694+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.694+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:33:58.694+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xf6ea 2019-12-07T19:33:58.694+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:33:58.694+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x6ea 2019-12-07T19:33:58.694+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.694+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb5b and padding block with 0x4a5 2019-12-07T19:33:58.694+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e819431e0 finish 2019-12-07T19:33:58.694+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.694+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2fb5b 2019-12-07T19:33:58.694+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.694+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.694+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.694+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.694+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000097134s (0.000000106s flush + 0.000097028s kv commit) 2019-12-07T19:33:58.694+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.694+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2fb5b~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa600> 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 kv_submitted 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa600 2019-12-07T19:33:58.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 finishing 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa600 onodes 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa600 done 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df900 q now empty 2019-12-07T19:33:58.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa600 [] 2019-12-07T19:33:58.695+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2fb5b 0x2fb5b~471 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xfb5b 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb5b 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xfcc and padding block with 0x34 2019-12-07T19:33:58.695+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e9f10 blob([0xb2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8000 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.695+0000 7fe2c5a62700 20 bluestore.BufferSpace(0x558e83af8018 in 0x558e80219260) _discard 0x0~1000 2019-12-07T19:33:58.695+0000 7fe2c5a62700 20 _add level 1 near 0 on buffer(0x558e81171290 space 0x558e83af8018 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.695+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 0x0~1000 = 4096 2019-12-07T19:33:58.695+0000 7fe2c5a62700 20 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.695+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: Getting attr on 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.695+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: not in cache 1:276d5164:::smithi13140796-22:head 2019-12-07T19:33:58.695+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:33:58.695+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] get_hash_info: found on disk, size 90112 2019-12-07T19:33:58.695+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# hinfo_key 2019-12-07T19:33:58.695+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# hinfo_key = 0 2019-12-07T19:33:58.695+0000 7fe2c5a62700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpReadReply(1.4s0 21/18 ECSubReadReply(tid=4, attrs_read=0)) v2 -- 0x558e81a03680 con 0x558e818fe000 2019-12-07T19:33:58.695+0000 7fe2c5a62700 10 osd.0 21 dequeue_op 0x558e83b0a9a0 finish 2019-12-07T19:33:58.695+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.695+0000 7fe2c9a6a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpWriteReply(1.4s0 21/18 ECSubWriteReply(tid=3, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e818a9000 con 0x558e818fe000 2019-12-07T19:33:58.695+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x2ffcc 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000283882s (0.000000099s flush + 0.000283783s kv commit) 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1200> 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:33:58.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:33:58.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:33:58.695+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.695+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=21, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e818ee600 con 0x558e818d7000 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.695+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.695+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 26 ==== MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=22, reqid=client.4540.0:77, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e83ac9000 con 0x558e818d7000 2019-12-07T19:33:58.695+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943760 prio 127 cost 0 latency 0.000039 epoch 21 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=22, reqid=client.4540.0:77, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.695+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=22, reqid=client.4540.0:77, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=22, reqid=client.4540.0:77, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=22, reqid=client.4540.0:77, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81943760 prio 127 cost 0 latency 0.000080 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=22, reqid=client.4540.0:77, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.fs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=22, reqid=client.4540.0:77, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'3], crt=21'3) [21'4 (21'3) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:77 2019-12-07T19:33:58.684306+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] add_log_entry 21'4 (21'3) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:77 2019-12-07T19:33:58.684306+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.695+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e8195fb00 seq 11 2019-12-07T19:33:58.695+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 3 keys 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 3 keys = 0 2019-12-07T19:33:58.695+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.695+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195fb00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195fb00 onodes 0x558e83b08f00 shared_blobs 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83b09008) update 1#1:f9bf9ba5:::smithi13140796-26:head# 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:f9bf9ba5:::smithi13140796-26:head# is 741 (485 bytes onode + 2 bytes spanning blobs + 254 bytes inline extents) 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195fb00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 prepare 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195fb00 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 io_done 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.695+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81943760 finish 2019-12-07T19:33:58.695+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 52 < min_flush_size 524288 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1792 < min_flush_size 524288 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b08f00 had 1 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1815 < min_flush_size 524288 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1815 < min_flush_size 524288 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x2ffcc~717 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x2ffcc 0x2ffcc~717 to file(ino 16 size 0x30000 mtime 2019-12-07T19:33:58.626995+0000 allocated 30000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000]) 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _allocate len 0x6e3 from 0 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _allocate len 0x6e3 from 1 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd140000 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd160000~10000/10000,10000,acd140000 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x40000 to include allocated 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 25 (was clean) 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd130000~10000 x_off 0xffcc 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xfcc 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x6e3 and padding block with 0x91d 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x40000 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x306e3 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (25) on file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]), flushing log 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 25 len 0x48 crc 0xa68b8675) 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x18000~1000 to file(ino 1 size 0x18000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.695+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x18000 0x18000~1000 to file(ino 1 size 0x18000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x19000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.695+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x18000 2019-12-07T19:33:58.696+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x19000 2019-12-07T19:33:58.696+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:58.696+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:33:58.696+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000022 2019-12-07T19:33:58.696+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.696+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 25 2019-12-07T19:33:58.696+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.696+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000471350s (0.000000073s flush + 0.000471277s kv commit) 2019-12-07T19:33:58.696+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195fb00> 2019-12-07T19:33:58.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.696+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 kv_submitted 2019-12-07T19:33:58.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195fb00 2019-12-07T19:33:58.696+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 finishing 2019-12-07T19:33:58.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195fb00 onodes 0x558e83b08f00 2019-12-07T19:33:58.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 done 2019-12-07T19:33:58.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:33:58.696+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195fb00 [] 2019-12-07T19:33:58.696+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.696+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.696+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=22, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558e811f6c00 con 0x558e818d7000 2019-12-07T19:33:58.696+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.697+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 16 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:82, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 116660+0+0 (unknown 0 0 0) 0x558e819d9800 con 0x558e818fe400 2019-12-07T19:33:58.697+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943ce0 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:82, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.697+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:82, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:82, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:82, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e81943ce0 prio 127 cost 0 latency 0.000077 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:82, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:82, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:82 2019-12-07T19:33:58.690760+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:82 2019-12-07T19:33:58.690760+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.697+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e83b28600 seq 9 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs3_head 0x558e819c41e0) get_onode oid 3#1:d07269f7:::smithi13140796-28:head# key 0x838000000000000001d07269f721736d'ithi13140796-28!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.697+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1094 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# = 0 2019-12-07T19:33:58.697+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# 2 keys 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# 2 keys = 0 2019-12-07T19:33:58.697+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.697+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# 0x0~1c000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:d07269f7:::smithi13140796-28:head# 0x0~1c000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1c000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83b28600 14 blobs 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate 0x1c000/1000,1c000,0 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate extent: 0xfb000~1c000/1000,1c000,0 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0xfb000~1c000] 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d36bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194f880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d36bd0 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f880 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d36bd0) get_ref 0x0~2000 Blob(0x558e83d36bd0 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f880 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d36bd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83d36bd0 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f880 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e8194f898 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d36ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d36f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d36ee0 blob([0xfd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36f50 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d36ee0) get_ref 0x0~2000 Blob(0x558e83d36ee0 blob([0xfd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36f50 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d36ee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83d36ee0 blob([0xfd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36f50 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d36f68 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d370a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d37110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d370a0 blob([0xff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d37110 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d370a0) get_ref 0x0~2000 Blob(0x558e83d370a0 blob([0xff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d37110 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d370a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83d370a0 blob([0xff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37110 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d37128 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d37180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d371f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d37180 blob([0x101000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d371f0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d37180) get_ref 0x0~2000 Blob(0x558e83d37180 blob([0x101000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d371f0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d37180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83d37180 blob([0x101000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d371f0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d37208 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d37260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d372d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d37260 blob([0x103000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d372d0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d37260) get_ref 0x0~2000 Blob(0x558e83d37260 blob([0x103000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d372d0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d37260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83d37260 blob([0x103000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d372d0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d372e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0ef50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0efc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0ef50 blob([0x105000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0efc0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0ef50) get_ref 0x0~2000 Blob(0x558e83d0ef50 blob([0x105000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0efc0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0ef50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83d0ef50 blob([0x105000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0efc0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0efd8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0f030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0f030 blob([0x107000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f0a0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f030) get_ref 0x0~2000 Blob(0x558e83d0f030 blob([0x107000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f0a0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83d0f030 blob([0x107000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f0a0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0f0b8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0f110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194e5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0f110 blob([0x109000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e5b0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f110) get_ref 0x0~2000 Blob(0x558e83d0f110 blob([0x109000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e5b0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83d0f110 blob([0x109000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e5b0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e8194e5c8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194fa40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81866bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194fa40 blob([0x10b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81866bd0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e8194fa40) get_ref 0x0~2000 Blob(0x558e8194fa40 blob([0x10b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81866bd0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e8194fa40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e8194fa40 blob([0x10b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81866bd0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81866be8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d373b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d37420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d373b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d37420 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d373b0) get_ref 0x0~2000 Blob(0x558e83d373b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d37420 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d373b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83d373b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37420 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d37438 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d37340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e9490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d37340 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9490 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d37340) get_ref 0x0~2000 Blob(0x558e83d37340 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9490 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d37340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83d37340 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9490 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e817e94a8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0f180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f1f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0f180 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f1f0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f180) get_ref 0x0~2000 Blob(0x558e83d0f180 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f1f0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83d0f180 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f1f0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0f208 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0f260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f2d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0f260 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f2d0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f260) get_ref 0x0~2000 Blob(0x558e83d0f260 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f2d0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83d0f260 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0f2e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0f340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f3b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0f340 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f3b0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f340) get_ref 0x0~2000 Blob(0x558e83d0f340 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f3b0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83d0f340 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f3b0 sbid 0x0)) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0f3c8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1c000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83cb2108) dirty_range mark inline shard dirty 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# 0x0~1c000 = 0 2019-12-07T19:33:58.697+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.697+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83b28600 cost 176310 (15 ios * 4000 + 116310 bytes) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83b28600 onodes 0x558e83cb2000 shared_blobs 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83cb2108) update 3#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83cb2108) update inline shard 269 bytes from 14 extents 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:d07269f7:::smithi13140796-28:head# is 688 (413 bytes onode + 2 bytes spanning blobs + 273 bytes inline extents) 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83b28600 allocated 0x[fb000~1c000] released 0x[] 2019-12-07T19:33:58.697+0000 7fe2c6263700 10 freelist allocate 0xfb000~1c000 2019-12-07T19:33:58.697+0000 7fe2c6263700 20 freelist _xor first_key 0x80000 last_key 0x100000 2019-12-07T19:33:58.698+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83b28600 prepare 2019-12-07T19:33:58.698+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83b28600 2019-12-07T19:33:58.698+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.698+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e81943ce0 finish 2019-12-07T19:33:58.698+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.698+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83b28600 aio_wait 2019-12-07T19:33:58.698+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83b28600 2019-12-07T19:33:58.698+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83b28600 io_done 2019-12-07T19:33:58.698+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.698+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.698+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.699+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1429 < min_flush_size 524288 2019-12-07T19:33:58.699+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2000 had 1 2019-12-07T19:33:58.699+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.699+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1452 < min_flush_size 524288 2019-12-07T19:33:58.699+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1452 < min_flush_size 524288 2019-12-07T19:33:58.699+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.699+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x306e3~5ac to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.699+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x306e3 0x306e3~5ac to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.699+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.699+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x6e3 2019-12-07T19:33:58.699+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x6e3 2019-12-07T19:33:58.699+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.699+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc8f and padding block with 0x371 2019-12-07T19:33:58.699+0000 7fe2e4535700 10 osd.0 21 tick 2019-12-07T19:33:58.699+0000 7fe2e4535700 10 osd.0 21 do_waiters -- start 2019-12-07T19:33:58.699+0000 7fe2e4535700 10 osd.0 21 do_waiters -- finish 2019-12-07T19:33:58.699+0000 7fe2e4535700 20 osd.0 21 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:58.699+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x30c8f 2019-12-07T19:33:58.699+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.699+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.699+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.699+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.699+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000973368s (0.000289284s flush + 0.000684084s kv commit) 2019-12-07T19:33:58.699+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83b28600> 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.699+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83b28600 kv_submitted 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83b28600 2019-12-07T19:33:58.699+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83b28600 finishing 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83b28600 onodes 0x558e83cb2000 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e94a8 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8fcb0 space 0x558e817e94a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81866be8 in 0x558e80219c00) _finish_write discard buffer(0x558e81171050 space 0x558e81866be8 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194e5c8 in 0x558e80219c00) _finish_write discard buffer(0x558e81170900 space 0x558e8194e5c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194f898 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8eb40 space 0x558e8194f898 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0efd8 in 0x558e80219c00) _finish_write discard buffer(0x558e81171b00 space 0x558e83d0efd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0f0b8 in 0x558e80219c00) _finish_write discard buffer(0x558e81170c60 space 0x558e83d0f0b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0f208 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8fdd0 space 0x558e83d0f208 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0f2e8 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8f170 space 0x558e83d0f2e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0f3c8 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8e480 space 0x558e83d0f3c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d36f68 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8ecf0 space 0x558e83d36f68 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=11, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e81954e00 con 0x558e818fe400 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d37128 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8e990 space 0x558e83d37128 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d37208 in 0x558e80219c00) _finish_write discard buffer(0x558e811715f0 space 0x558e83d37208 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d372e8 in 0x558e80219c00) _finish_write discard buffer(0x558e81170cf0 space 0x558e83d372e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d37438 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8fc20 space 0x558e83d37438 0x0~2000 writing nocache) 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83b28600 done 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:58.699+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83b28600 [] 2019-12-07T19:33:58.699+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.699+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.699+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.700+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 8 ==== osd_map(19..21 src has 1..21) v4 ==== 1883+0+0 (unknown 0 0 0) 0x558e8123b440 con 0x558e818d6000 2019-12-07T19:33:58.700+0000 7fe2d4a80700 20 osd.0 21 OSD::ms_dispatch: osd_map(19..21 src has 1..21) v4 2019-12-07T19:33:58.700+0000 7fe2d4a80700 10 osd.0 21 do_waiters -- start 2019-12-07T19:33:58.700+0000 7fe2d4a80700 10 osd.0 21 do_waiters -- finish 2019-12-07T19:33:58.700+0000 7fe2d4a80700 20 osd.0 21 _dispatch 0x558e8123b440 osd_map(19..21 src has 1..21) v4 2019-12-07T19:33:58.700+0000 7fe2d4a80700 3 osd.0 21 handle_osd_map epochs [19,21], i have 21, src has [1,21] 2019-12-07T19:33:58.700+0000 7fe2d4a80700 10 osd.0 21 no new maps here, dropping 2019-12-07T19:33:58.700+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 9 ==== MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:88, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 124852+0+0 (unknown 0 0 0) 0x558e83cbd000 con 0x558e818d6000 2019-12-07T19:33:58.700+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0a000 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:88, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.700+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:88, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:88, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:88, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e21) pg 0x558e819e7000 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 osd.0 21 dequeue_op 0x558e83b0a000 prio 127 cost 0 latency 0.000106 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:88, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 osd.0 21 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 19 -> 21 (as per caller) 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:88, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 crt=0'0 active mbc={}] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:88 2019-12-07T19:33:58.696853+0000 0 ObjectCleanRegions clean_offsets: [491520~18446744073709060095], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] add_log_entry 21'1 (0'0) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:88 2019-12-07T19:33:58.696853+0000 0 ObjectCleanRegions clean_offsets: [491520~18446744073709060095], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.700+0000 7fe2c5261700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e819a1500 seq 5 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.ds5_head 0x558e817b5860) get_onode oid 5#1:bca18876:::smithi13140796-30:head# key 0x858000000000000001bca188'v!smithi13140796-30!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.700+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.ds5_head 5#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1095 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.ds5_head 5#1:bca18876:::smithi13140796-30:head# = 0 2019-12-07T19:33:58.700+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.ds5_head 5#1:bca18876:::smithi13140796-30:head# 2 keys 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.ds5_head 5#1:bca18876:::smithi13140796-30:head# 2 keys = 0 2019-12-07T19:33:58.700+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.ds5_head 5#1:bca18876:::smithi13140796-30:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.ds5_head 5#1:bca18876:::smithi13140796-30:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.700+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.ds5_head 5#1:bca18876:::smithi13140796-30:head# 0x0~1e000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#1:bca18876:::smithi13140796-30:head# 0x0~1e000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1e000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a1500 15 blobs 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 fbmap_alloc 0x558e80233300 allocate 0x1e000/1000,1e000,0 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x117000~1e000/1000,1e000,0 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x117000~1e000] 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194e3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fe30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194e3f0 blob([0x117000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fe30 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e8194e3f0) get_ref 0x0~2000 Blob(0x558e8194e3f0 blob([0x117000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fe30 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e8194e3f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e8194e3f0 blob([0x117000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fe30 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83d0fe48 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194e070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194e380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194e070 blob([0x119000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e380 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e8194e070) get_ref 0x0~2000 Blob(0x558e8194e070 blob([0x119000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e380 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e8194e070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e8194e070 blob([0x119000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e380 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e8194e398 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194e460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e9dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194e460 blob([0x11b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9dc0 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e8194e460) get_ref 0x0~2000 Blob(0x558e8194e460 blob([0x11b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9dc0 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e8194e460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e8194e460 blob([0x11b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9dc0 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e817e9dd8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc4000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc4000 blob([0x11d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4070 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc4000) get_ref 0x0~2000 Blob(0x558e83cc4000 blob([0x11d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4070 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc4000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83cc4000 blob([0x11d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc4070 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc4088 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc40e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc40e0 blob([0x11f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4150 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc40e0) get_ref 0x0~2000 Blob(0x558e83cc40e0 blob([0x11f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4150 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc40e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83cc40e0 blob([0x11f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc4150 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc4168 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc41c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc41c0 blob([0x121000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4230 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc41c0) get_ref 0x0~2000 Blob(0x558e83cc41c0 blob([0x121000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4230 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc41c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83cc41c0 blob([0x121000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc4230 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc4248 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc42a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc42a0 blob([0x123000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4310 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc42a0) get_ref 0x0~2000 Blob(0x558e83cc42a0 blob([0x123000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4310 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc42a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83cc42a0 blob([0x123000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc4310 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc4328 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc4380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc43f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc4380 blob([0x125000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc43f0 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc4380) get_ref 0x0~2000 Blob(0x558e83cc4380 blob([0x125000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc43f0 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc4380) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83cc4380 blob([0x125000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc43f0 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc4408 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc4460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc44d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc4460 blob([0x127000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc44d0 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc4460) get_ref 0x0~2000 Blob(0x558e83cc4460 blob([0x127000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc44d0 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc4460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83cc4460 blob([0x127000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc44d0 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc44e8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc4540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc45b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc4540 blob([0x129000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc45b0 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc4540) get_ref 0x0~2000 Blob(0x558e83cc4540 blob([0x129000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc45b0 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc4540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83cc4540 blob([0x129000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc45b0 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc45c8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc4620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc4620 blob([0x12b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4690 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc4620) get_ref 0x0~2000 Blob(0x558e83cc4620 blob([0x12b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4690 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc4620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83cc4620 blob([0x12b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc4690 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc46a8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc4700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc4700 blob([0x12d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4770 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc4700) get_ref 0x0~2000 Blob(0x558e83cc4700 blob([0x12d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4770 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc4700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83cc4700 blob([0x12d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc4770 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc4788 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc47e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc47e0 blob([0x12f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4850 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc47e0) get_ref 0x0~2000 Blob(0x558e83cc47e0 blob([0x12f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4850 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc47e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83cc47e0 blob([0x12f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc4850 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc4868 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc48c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc48c0 blob([0x131000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4930 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc48c0) get_ref 0x0~2000 Blob(0x558e83cc48c0 blob([0x131000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4930 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc48c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83cc48c0 blob([0x131000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc4930 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc4948 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc49a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc49a0 blob([0x133000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4a10 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc49a0) get_ref 0x0~2000 Blob(0x558e83cc49a0 blob([0x133000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc4a10 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc49a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e83cc49a0 blob([0x133000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc4a10 sbid 0x0)) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc4a28 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1e000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.extentmap(0x558e83cb2b08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.ds5_head 5#1:bca18876:::smithi13140796-30:head# 0x0~1e000 = 0 2019-12-07T19:33:58.700+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.ds5_head 5#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.ds5_head 5#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.700+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1500 cost 189310 (16 ios * 4000 + 125310 bytes) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1500 onodes 0x558e83cb2a00 shared_blobs 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.extentmap(0x558e83cb2b08) update 5#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore.extentmap(0x558e83cb2b08) update inline shard 288 bytes from 15 extents 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:bca18876:::smithi13140796-30:head# is 707 (413 bytes onode + 2 bytes spanning blobs + 292 bytes inline extents) 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1500 allocated 0x[117000~1e000] released 0x[] 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 freelist allocate 0x117000~1e000 2019-12-07T19:33:58.700+0000 7fe2c5261700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 prepare 2019-12-07T19:33:58.700+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e819a1500 2019-12-07T19:33:58.701+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 active mbc={}] op_applied version 21'1 2019-12-07T19:33:58.701+0000 7fe2c5261700 10 osd.0 21 dequeue_op 0x558e83b0a000 finish 2019-12-07T19:33:58.701+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.701+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 aio_wait 2019-12-07T19:33:58.701+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1500 2019-12-07T19:33:58.701+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 io_done 2019-12-07T19:33:58.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.701+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2233 < min_flush_size 524288 2019-12-07T19:33:58.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2a00 had 1 2019-12-07T19:33:58.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:33:58.701+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2256 < min_flush_size 524288 2019-12-07T19:33:58.701+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2256 < min_flush_size 524288 2019-12-07T19:33:58.701+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.701+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x30c8f~8d0 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.701+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x30c8f 0x30c8f~8d0 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.701+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.701+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xc8f 2019-12-07T19:33:58.701+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc8f 2019-12-07T19:33:58.701+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.701+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x55f and padding block with 0xaa1 2019-12-07T19:33:58.701+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3155f 2019-12-07T19:33:58.701+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.701+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.701+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.701+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000157495s (0.000036523s flush + 0.000120972s kv commit) 2019-12-07T19:33:58.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1500> 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.701+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 kv_submitted 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1500 2019-12-07T19:33:58.701+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 finishing 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1500 onodes 0x558e83cb2a00 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e9dd8 in 0x558e80219dc0) _finish_write discard buffer(0x558e81171b00 space 0x558e817e9dd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194e398 in 0x558e80219dc0) _finish_write discard buffer(0x558e81170900 space 0x558e8194e398 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc4088 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8eab0 space 0x558e83cc4088 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc4168 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8ea20 space 0x558e83cc4168 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc4248 in 0x558e80219dc0) _finish_write discard buffer(0x558e81170c60 space 0x558e83cc4248 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc4328 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8e510 space 0x558e83cc4328 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc4408 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8e5a0 space 0x558e83cc4408 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc44e8 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8f320 space 0x558e83cc44e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc45c8 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8f3b0 space 0x558e83cc45c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc46a8 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8f440 space 0x558e83cc46a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc4788 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8f4d0 space 0x558e83cc4788 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc4868 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8f560 space 0x558e83cc4868 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc4948 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8e3f0 space 0x558e83cc4948 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc4a28 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8f290 space 0x558e83cc4a28 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0fe48 in 0x558e80219dc0) _finish_write discard buffer(0x558e81171050 space 0x558e83d0fe48 0x0~2000 writing nocache) 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1500 done 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:33:58.701+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1500 [] 2019-12-07T19:33:58.701+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.701+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=8, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e81955400 con 0x558e818d6000 2019-12-07T19:33:58.701+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.701+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.703+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 55 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83cbe800 con 0x558e818d7c00 2019-12-07T19:33:58.703+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81942160 prio 127 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.703+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.703+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.703+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.703+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.703+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.703+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81942160 prio 127 cost 0 latency 0.000077 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.7s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.703+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=45, reqid=client.4540.0:73, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.703+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'5], crt=21'4) [] 2019-12-07T19:33:58.703+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'5 (0'0) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:73 2019-12-07T19:33:58.681506+0000 0 ObjectCleanRegions clean_offsets: [409600~18446744073709142015], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.703+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.703+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.703+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.703+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.703+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.703+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e83c56600 seq 14 2019-12-07T19:33:58.703+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.703+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.703+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.703+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.703+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.703+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.703+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56600 onodes shared_blobs 2019-12-07T19:33:58.703+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.703+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 prepare 2019-12-07T19:33:58.703+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56600 2019-12-07T19:33:58.703+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 io_done 2019-12-07T19:33:58.703+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81942160 finish 2019-12-07T19:33:58.703+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.703+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.703+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.703+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.703+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.703+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.703+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.703+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.703+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3155f~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.703+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3155f 0x3155f~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.703+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.703+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x155f 2019-12-07T19:33:58.703+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x55f 2019-12-07T19:33:58.703+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.703+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x9d0 and padding block with 0x630 2019-12-07T19:33:58.703+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x319d0 2019-12-07T19:33:58.703+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.703+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.703+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.703+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.703+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000124984s (0.000000126s flush + 0.000124858s kv commit) 2019-12-07T19:33:58.703+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.703+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.703+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56600> 2019-12-07T19:33:58.703+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.703+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 kv_submitted 2019-12-07T19:33:58.703+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56600 2019-12-07T19:33:58.703+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 finishing 2019-12-07T19:33:58.703+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56600 onodes 2019-12-07T19:33:58.703+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56600 done 2019-12-07T19:33:58.703+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.703+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56600 [] 2019-12-07T19:33:58.703+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.703+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.703+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=45, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558e81239800 con 0x558e818d7c00 2019-12-07T19:33:58.703+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.704+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.704+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 56 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e83cbf000 con 0x558e818d7c00 2019-12-07T19:33:58.704+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81943b80 prio 127 cost 0 latency 0.000033 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.704+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81943b80 prio 127 cost 0 latency 0.000091 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.7s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=46, reqid=client.4540.0:74, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'5], crt=21'5) [21'6 (21'5) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:74 2019-12-07T19:33:58.681620+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'5) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:74 2019-12-07T19:33:58.681620+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.704+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e8195e300 seq 15 2019-12-07T19:33:58.704+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 3 keys 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 3 keys = 0 2019-12-07T19:33:58.704+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.704+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e300 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e300 onodes 0x558e83b09900 shared_blobs 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83b09a08) update 3#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:ee170468:::smithi13140796-25:head# is 738 (485 bytes onode + 2 bytes spanning blobs + 251 bytes inline extents) 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 prepare 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e300 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 io_done 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.704+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81943b80 finish 2019-12-07T19:33:58.704+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.704+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 27 ==== MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=24, reqid=client.4540.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e819d8800 con 0x558e818d7000 2019-12-07T19:33:58.704+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81942c60 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=24, reqid=client.4540.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.704+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=24, reqid=client.4540.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.704+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.704+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.704+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1778 < min_flush_size 524288 2019-12-07T19:33:58.704+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b09900 had 1 2019-12-07T19:33:58.704+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.704+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1801 < min_flush_size 524288 2019-12-07T19:33:58.704+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1801 < min_flush_size 524288 2019-12-07T19:33:58.704+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.704+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x319d0~709 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.704+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x319d0 0x319d0~709 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.704+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.704+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x19d0 2019-12-07T19:33:58.704+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x9d0 2019-12-07T19:33:58.704+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.704+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd9 and padding block with 0xf27 2019-12-07T19:33:58.704+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=24, reqid=client.4540.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.704+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.704+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=24, reqid=client.4540.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.704+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81942c60 prio 127 cost 0 latency 0.000152 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=24, reqid=client.4540.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.704+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=24, reqid=client.4540.0:77, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.704+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'4], crt=21'3) [] 2019-12-07T19:33:58.704+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] rollforward: entry=21'4 (21'3) modify 1:f9bf9ba5:::smithi13140796-26:head by client.4540.0:77 2019-12-07T19:33:58.684306+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.704+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.704+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.704+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.704+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.704+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:58.704+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e83c56900 seq 12 2019-12-07T19:33:58.704+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:58.704+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:58.704+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.704+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.704+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:58.704+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.704+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56900 onodes shared_blobs 2019-12-07T19:33:58.704+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.704+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 prepare 2019-12-07T19:33:58.704+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56900 2019-12-07T19:33:58.704+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 io_done 2019-12-07T19:33:58.704+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81942c60 finish 2019-12-07T19:33:58.704+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.704+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x320d9 2019-12-07T19:33:58.704+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.704+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.704+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.704+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.705+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000241086s (0.000000109s flush + 0.000240977s kv commit) 2019-12-07T19:33:58.705+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.705+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.705+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:58.705+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.705+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.705+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.705+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x320d9~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.705+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x320d9 0x320d9~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.705+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.705+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x20d9 2019-12-07T19:33:58.705+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd9 2019-12-07T19:33:58.705+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.705+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x54a and padding block with 0xab6 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e300> 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.705+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 kv_submitted 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e300 2019-12-07T19:33:58.705+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 finishing 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e300 onodes 0x558e83b09900 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 done 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.705+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e300 [] 2019-12-07T19:33:58.705+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.705+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=46, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558e81964a00 con 0x558e818d7c00 2019-12-07T19:33:58.705+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.705+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3254a 2019-12-07T19:33:58.705+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.705+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.705+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.705+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.705+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000143999s (0.000000098s flush + 0.000143901s kv commit) 2019-12-07T19:33:58.705+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56900> 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.705+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 kv_submitted 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56900 2019-12-07T19:33:58.705+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 finishing 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56900 onodes 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56900 done 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:33:58.705+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56900 [] 2019-12-07T19:33:58.705+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.705+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.705+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=24, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558e818efc00 con 0x558e818d7000 2019-12-07T19:33:58.705+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 28 ==== MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=25, to_read={1:f9bf9ba5:::smithi13140796-26:head=0,4096,0}, subchunks={1:f9bf9ba5:::smithi13140796-26:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e83c72280 con 0x558e818d7000 2019-12-07T19:33:58.705+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.705+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a2c0 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=25, to_read={1:f9bf9ba5:::smithi13140796-26:head=0,4096,0}, subchunks={1:f9bf9ba5:::smithi13140796-26:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.705+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=25, to_read={1:f9bf9ba5:::smithi13140796-26:head=0,4096,0}, subchunks={1:f9bf9ba5:::smithi13140796-26:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=25, to_read={1:f9bf9ba5:::smithi13140796-26:head=0,4096,0}, subchunks={1:f9bf9ba5:::smithi13140796-26:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=25, to_read={1:f9bf9ba5:::smithi13140796-26:head=0,4096,0}, subchunks={1:f9bf9ba5:::smithi13140796-26:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.705+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83b0a2c0 prio 127 cost 0 latency 0.000081 MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=25, to_read={1:f9bf9ba5:::smithi13140796-26:head=0,4096,0}, subchunks={1:f9bf9ba5:::smithi13140796-26:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.705+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=25, to_read={1:f9bf9ba5:::smithi13140796-26:head=0,4096,0}, subchunks={1:f9bf9ba5:::smithi13140796-26:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.705+0000 7fe2c425f700 25 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.705+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x0~1000 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x1a000 (106496) 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ec40 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ec40 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.705+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ec40 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e8194ec58 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e83c8f710 space 0x558e8194ec58 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.705+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x0~1000 = 4096 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.705+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: Getting attr on 1:f9bf9ba5:::smithi13140796-26:head 2019-12-07T19:33:58.705+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: not in cache 1:f9bf9ba5:::smithi13140796-26:head 2019-12-07T19:33:58.705+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 2019-12-07T19:33:58.705+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: found on disk, size 106496 2019-12-07T19:33:58.705+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# hinfo_key 2019-12-07T19:33:58.705+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# hinfo_key = 0 2019-12-07T19:33:58.705+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.fs0 21/18 ECSubReadReply(tid=25, attrs_read=0)) v2 -- 0x558e83c72780 con 0x558e818d7000 2019-12-07T19:33:58.705+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83b0a2c0 finish 2019-12-07T19:33:58.705+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.708+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 17 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:82, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e819d8000 con 0x558e818fe400 2019-12-07T19:33:58.708+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a840 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:82, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.708+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:82, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.708+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 18 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=15, reqid=client.4540.0:83, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e819db800 con 0x558e818fe400 2019-12-07T19:33:58.708+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a9a0 prio 127 cost 0 latency 0.000046 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=15, reqid=client.4540.0:83, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.708+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=15, reqid=client.4540.0:83, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:82, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:82, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e83b0a840 prio 127 cost 0 latency 0.000956 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:82, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.bs3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:82, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [] 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:82 2019-12-07T19:33:58.690760+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.709+0000 7fe2ca26b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e81a0b800 seq 10 2019-12-07T19:33:58.709+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b800 onodes shared_blobs 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 prepare 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b800 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 io_done 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e83b0a840 finish 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=15, reqid=client.4540.0:83, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=15, reqid=client.4540.0:83, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e83b0a9a0 prio 127 cost 0 latency 0.000415 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=15, reqid=client.4540.0:83, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.bs3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=15, reqid=client.4540.0:83, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'3], crt=21'3) [21'4 (21'3) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:83 2019-12-07T19:33:58.690784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] add_log_entry 21'4 (21'3) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:83 2019-12-07T19:33:58.690784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.709+0000 7fe2ca26b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e8195f500 seq 11 2019-12-07T19:33:58.709+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# 3 keys 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# 3 keys = 0 2019-12-07T19:33:58.709+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.709+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195f500 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195f500 onodes 0x558e83cb2000 shared_blobs 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e83cb2108) update 3#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:d07269f7:::smithi13140796-28:head# is 760 (485 bytes onode + 2 bytes spanning blobs + 273 bytes inline extents) 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195f500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 prepare 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195f500 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 io_done 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.709+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e83b0a9a0 finish 2019-12-07T19:33:58.709+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.709+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.709+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.709+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.709+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 2 2019-12-07T19:33:58.709+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2914 < min_flush_size 524288 2019-12-07T19:33:58.709+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2000 had 1 2019-12-07T19:33:58.709+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.709+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2937 < min_flush_size 524288 2019-12-07T19:33:58.709+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2937 < min_flush_size 524288 2019-12-07T19:33:58.709+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.709+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3254a~b79 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.709+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3254a 0x3254a~b79 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.709+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.709+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x254a 2019-12-07T19:33:58.709+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x54a 2019-12-07T19:33:58.709+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.709+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc3 and padding block with 0xf3d 2019-12-07T19:33:58.709+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 29 ==== MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=26, reqid=client.4540.0:97, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 137140+0+0 (unknown 0 0 0) 0x558e811b9000 con 0x558e818d7000 2019-12-07T19:33:58.709+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81942420 prio 127 cost 0 latency 0.000038 epoch 21 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=26, reqid=client.4540.0:97, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.709+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=26, reqid=client.4540.0:97, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.709+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.709+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=26, reqid=client.4540.0:97, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.709+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.709+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=26, reqid=client.4540.0:97, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.709+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81942420 prio 127 cost 0 latency 0.000133 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=26, reqid=client.4540.0:97, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.709+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=26, reqid=client.4540.0:97, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.709+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'4], crt=21'4) [21'5 (0'0) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:97 2019-12-07T19:33:58.707663+0000 0 ObjectCleanRegions clean_offsets: [540672~18446744073709010943], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.709+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] add_log_entry 21'5 (0'0) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:97 2019-12-07T19:33:58.707663+0000 0 ObjectCleanRegions clean_offsets: [540672~18446744073709010943], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.709+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.709+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.709+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.710+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e811fac00 seq 13 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) get_onode oid 1#1:fc3a86a2:::smithi13140796-33:head# key 0x818000000000000001fc3a86a221736d'ithi13140796-33!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.710+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1096 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# = 0 2019-12-07T19:33:58.710+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 2 keys 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 2 keys = 0 2019-12-07T19:33:58.710+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.710+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x0~21000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 1#1:fc3a86a2:::smithi13140796-33:head# 0x0~21000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~21000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fac00 17 blobs 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 fbmap_alloc 0x558e80233300 allocate 0x21000/1000,21000,0 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x135000~21000/1000,21000,0 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x135000~21000] 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac5e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194f960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f960 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83ac5e30) get_ref 0x0~2000 Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f960 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83ac5e30) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f960 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e8194f978 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac4d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ebd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ebd0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83ac4d20) get_ref 0x0~2000 Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ebd0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83ac4d20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ebd0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0ebe8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af9b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0eb60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0eb60 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9b20) get_ref 0x0~2000 Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0eb60 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83af9b20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0eb60 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0eb78 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0ec40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ecb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ecb0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0ec40) get_ref 0x0~2000 Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ecb0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0ec40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ecb0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0ecc8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0ed20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ed90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ed90 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0ed20) get_ref 0x0~2000 Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ed90 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0ed20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ed90 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0eda8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0ee00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ee70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ee70 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0ee00) get_ref 0x0~2000 Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ee70 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0ee00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ee70 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0ee88 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0eee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc53b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc53b0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0eee0) get_ref 0x0~2000 Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc53b0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0eee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc53b0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc53c8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc5420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5490 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc5420) get_ref 0x0~2000 Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5490 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc5420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5490 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc54a8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc5500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5570 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc5500) get_ref 0x0~2000 Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5570 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc5500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5570 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5588 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc55e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5650 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc55e0) get_ref 0x0~2000 Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5650 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc55e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5650 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5668 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc56c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5730 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc56c0) get_ref 0x0~2000 Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5730 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc56c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5730 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5748 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc57a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5810 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc57a0) get_ref 0x0~2000 Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5810 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc57a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5810 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5828 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc5880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc58f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc58f0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc5880) get_ref 0x0~2000 Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc58f0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc5880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc58f0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5908 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc5960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc59d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc59d0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc5960) get_ref 0x0~2000 Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc59d0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc5960) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc59d0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc59e8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc5a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5ab0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc5a40) get_ref 0x0~2000 Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5ab0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc5a40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5ab0 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5ac8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc5b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194e770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e770 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc5b20) get_ref 0x0~2000 Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e770 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc5b20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e770 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e8194e788 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc5b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc5b90 blob([0x155000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5c00 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc5b90) get_ref 0x0~1000 Blob(0x558e83cc5b90 blob([0x155000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5c00 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.blob(0x558e83cc5b90) get_ref init 0x1000, 1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~1000: 0x0~1000 Blob(0x558e83cc5b90 blob([0x155000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83cc5c00 sbid 0x0)) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5c18 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x21000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83b09c88) dirty_range mark inline shard dirty 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x0~21000 = 0 2019-12-07T19:33:58.710+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.710+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fac00 cost 208790 (18 ios * 4000 + 136790 bytes) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fac00 onodes 0x558e83b09b80 shared_blobs 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83b09c88) update 1#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83b09c88) update inline shard 323 bytes from 17 extents 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:fc3a86a2:::smithi13140796-33:head# is 742 (413 bytes onode + 2 bytes spanning blobs + 327 bytes inline extents) 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fac00 allocated 0x[135000~21000] released 0x[] 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 freelist allocate 0x135000~21000 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 prepare 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e811fac00 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.710+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81942420 finish 2019-12-07T19:33:58.710+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.711+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x330c3 2019-12-07T19:33:58.711+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.711+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.711+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.711+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.711+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.001932503s (0.000000241s flush + 0.001932262s kv commit) 2019-12-07T19:33:58.711+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0b800,0x558e8195f500> 2019-12-07T19:33:58.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.711+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 kv_submitted 2019-12-07T19:33:58.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b800 2019-12-07T19:33:58.711+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 finishing 2019-12-07T19:33:58.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b800 onodes 2019-12-07T19:33:58.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 done 2019-12-07T19:33:58.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f500 kv_submitted 2019-12-07T19:33:58.711+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b800 [] 2019-12-07T19:33:58.711+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.711+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 kv_submitted 2019-12-07T19:33:58.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195f500 2019-12-07T19:33:58.711+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 finishing 2019-12-07T19:33:58.711+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195f500 onodes 0x558e83cb2000 2019-12-07T19:33:58.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f500 done 2019-12-07T19:33:58.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:58.711+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195f500 [] 2019-12-07T19:33:58.711+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.711+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.711+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=13, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e811f6c00 con 0x558e818fe400 2019-12-07T19:33:58.711+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=15, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558e818a8c00 con 0x558e818fe400 2019-12-07T19:33:58.711+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.712+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 aio_wait 2019-12-07T19:33:58.712+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fac00 2019-12-07T19:33:58.712+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 io_done 2019-12-07T19:33:58.712+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.712+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.712+0000 7fe2e6d3a700 0 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558e818fe400 msgr2=0x558e818e5600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2019-12-07T19:33:58.712+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.712+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558e818fe400 msgr2=0x558e818e5600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 93 2019-12-07T19:33:58.712+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558e818fe400 msgr2=0x558e818e5600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.712+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558e818fe400 0x558e818e5600 unknown :-1 s=READY pgs=6 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.712+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558e818fe400 0x558e818e5600 unknown :-1 s=READY pgs=6 cs=0 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:33:58.712+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1454 < min_flush_size 524288 2019-12-07T19:33:58.712+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b09b80 had 1 2019-12-07T19:33:58.712+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:58.712+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1477 < min_flush_size 524288 2019-12-07T19:33:58.712+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1477 < min_flush_size 524288 2019-12-07T19:33:58.712+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.712+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x330c3~5c5 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.712+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x330c3 0x330c3~5c5 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.712+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.712+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x30c3 2019-12-07T19:33:58.712+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc3 2019-12-07T19:33:58.712+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.712+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x688 and padding block with 0x978 2019-12-07T19:33:58.712+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x33688 2019-12-07T19:33:58.712+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.712+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.712+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.712+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.712+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000201576s (0.000068821s flush + 0.000132755s kv commit) 2019-12-07T19:33:58.712+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fac00> 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.712+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 kv_submitted 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fac00 2019-12-07T19:33:58.712+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 finishing 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fac00 onodes 0x558e83b09b80 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194e788 in 0x558e8102e000) _finish_write discard buffer(0x558e81171b00 space 0x558e8194e788 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194f978 in 0x558e8102e000) _finish_write discard buffer(0x558e83d10090 space 0x558e8194f978 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc53c8 in 0x558e8102e000) _finish_write discard buffer(0x558e83d103f0 space 0x558e83cc53c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc54a8 in 0x558e8102e000) _finish_write discard buffer(0x558e83d10480 space 0x558e83cc54a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc5588 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8f170 space 0x558e83cc5588 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc5668 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8e480 space 0x558e83cc5668 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc5748 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8ecf0 space 0x558e83cc5748 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc5828 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8e990 space 0x558e83cc5828 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc5908 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8fc20 space 0x558e83cc5908 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc59e8 in 0x558e8102e000) _finish_write discard buffer(0x558e83c8eab0 space 0x558e83cc59e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc5ac8 in 0x558e8102e000) _finish_write discard buffer(0x558e81170cf0 space 0x558e83cc5ac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc5c18 in 0x558e8102e000) _finish_write discard buffer(0x558e81170900 space 0x558e83cc5c18 0x0~1000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0eb78 in 0x558e8102e000) _finish_write discard buffer(0x558e83d101b0 space 0x558e83d0eb78 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0ebe8 in 0x558e8102e000) _finish_write discard buffer(0x558e83d10120 space 0x558e83d0ebe8 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0ecc8 in 0x558e8102e000) _finish_write discard buffer(0x558e83d10240 space 0x558e83d0ecc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0eda8 in 0x558e8102e000) _finish_write discard buffer(0x558e83d102d0 space 0x558e83d0eda8 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0ee88 in 0x558e8102e000) _finish_write discard buffer(0x558e83d10360 space 0x558e83d0ee88 0x0~2000 writing nocache) 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fac00 done 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:33:58.712+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fac00 [] 2019-12-07T19:33:58.712+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.712+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.712+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.712+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=26, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558e81239e00 con 0x558e818d7000 2019-12-07T19:33:58.712+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.713+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 57 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 141236+0+0 (unknown 0 0 0) 0x558e83cbf800 con 0x558e818d7c00 2019-12-07T19:33:58.713+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81942dc0 prio 127 cost 0 latency 0.000033 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.713+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.713+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81942dc0 prio 127 cost 0 latency 0.000082 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.7s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.713+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=49, reqid=client.4540.0:100, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.713+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'6], crt=21'5) [21'7 (0'0) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:100 2019-12-07T19:33:58.711580+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.713+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] add_log_entry 21'7 (0'0) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:100 2019-12-07T19:33:58.711580+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.713+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.713+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.713+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.713+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.713+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e83c56300 seq 16 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.7s3_head 0x558e8122c5a0) get_onode oid 3#1:e7f24557:::smithi13140796-34:head# key 0x838000000000000001e7f245'W!smithi13140796-34!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.713+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1097 2019-12-07T19:33:58.713+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# = 0 2019-12-07T19:33:58.713+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# 2 keys 2019-12-07T19:33:58.713+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# 2 keys = 0 2019-12-07T19:33:58.713+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.713+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.713+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# 0x0~22000 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:e7f24557:::smithi13140796-34:head# 0x0~22000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.713+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~22000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83c56300 17 blobs 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.713+0000 7fe2c425f700 10 fbmap_alloc 0x558e80233300 allocate 0x22000/1000,22000,0 2019-12-07T19:33:58.713+0000 7fe2c425f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x156000~22000/1000,22000,0 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x156000~22000] 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c65030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c65030 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5f10 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore.blob(0x558e83c65030) get_ref 0x0~2000 Blob(0x558e83c65030 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5f10 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore.blob(0x558e83c65030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83c65030 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5f10 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5f28 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d37ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d37f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d37ea0 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d37f10 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37ea0) get_ref 0x0~2000 Blob(0x558e83d37ea0 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d37f10 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37ea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83d37ea0 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37f10 sbid 0x0)) 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d37f28 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.713+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d37f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d37f80 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03180 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37f80) get_ref 0x0~2000 Blob(0x558e83d37f80 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03180 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37f80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83d37f80 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03180 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03198 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d031f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d031f0 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03260 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d031f0) get_ref 0x0~2000 Blob(0x558e83d031f0 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03260 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d031f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83d031f0 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03260 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03278 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d032d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d032d0 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03340 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d032d0) get_ref 0x0~2000 Blob(0x558e83d032d0 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03340 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d032d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83d032d0 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03340 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03358 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d033b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d033b0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03420 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d033b0) get_ref 0x0~2000 Blob(0x558e83d033b0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03420 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d033b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83d033b0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03420 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03438 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03490 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03500 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03490) get_ref 0x0~2000 Blob(0x558e83d03490 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03500 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83d03490 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03518 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d035e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03570 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d035e0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03570) get_ref 0x0~2000 Blob(0x558e83d03570 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d035e0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83d03570 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d035e0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d035f8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d036c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03650 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d036c0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03650) get_ref 0x0~2000 Blob(0x558e83d03650 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d036c0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03650) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83d03650 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d036c0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d036d8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d037a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03730 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d037a0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03730) get_ref 0x0~2000 Blob(0x558e83d03730 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d037a0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83d03730 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d037a0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d037b8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03810 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03880 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03810) get_ref 0x0~2000 Blob(0x558e83d03810 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03880 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83d03810 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03880 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03898 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d038f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d038f0 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03960 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d038f0) get_ref 0x0~2000 Blob(0x558e83d038f0 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03960 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d038f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83d038f0 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03960 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03978 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d039d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d039d0 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03a40 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d039d0) get_ref 0x0~2000 Blob(0x558e83d039d0 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03a40 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d039d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83d039d0 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03a40 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03a58 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03ab0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03b20 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03ab0) get_ref 0x0~2000 Blob(0x558e83d03ab0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03b20 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83d03ab0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03b20 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03b38 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03b90 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03c00 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03b90) get_ref 0x0~2000 Blob(0x558e83d03b90 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03c00 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03b90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e83d03b90 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03c00 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03c18 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03c70 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03ce0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03c70) get_ref 0x0~2000 Blob(0x558e83d03c70 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03ce0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03c70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e83d03c70 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03ce0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03cf8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03d50 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03dc0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03d50) get_ref 0x0~2000 Blob(0x558e83d03d50 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03dc0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e83d03d50 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03dc0 sbid 0x0)) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03dd8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x22000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83b09288) dirty_range mark inline shard dirty 2019-12-07T19:33:58.714+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# 0x0~22000 = 0 2019-12-07T19:33:58.714+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.714+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.714+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.714+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.714+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56300 cost 212886 (18 ios * 4000 + 140886 bytes) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56300 onodes 0x558e83b09180 shared_blobs 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83b09288) update 3#1:e7f24557:::smithi13140796-34:head# 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83b09288) update inline shard 326 bytes from 17 extents 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:e7f24557:::smithi13140796-34:head# is 745 (413 bytes onode + 2 bytes spanning blobs + 330 bytes inline extents) 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56300 allocated 0x[156000~22000] released 0x[] 2019-12-07T19:33:58.714+0000 7fe2c425f700 10 freelist allocate 0x156000~22000 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:33:58.714+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 prepare 2019-12-07T19:33:58.714+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83c56300 2019-12-07T19:33:58.714+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] op_applied version 21'7 2019-12-07T19:33:58.714+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81942dc0 finish 2019-12-07T19:33:58.714+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.714+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 aio_wait 2019-12-07T19:33:58.714+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56300 2019-12-07T19:33:58.714+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 io_done 2019-12-07T19:33:58.714+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.714+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.714+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.714+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1457 < min_flush_size 524288 2019-12-07T19:33:58.714+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b09180 had 1 2019-12-07T19:33:58.714+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.714+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1480 < min_flush_size 524288 2019-12-07T19:33:58.714+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1480 < min_flush_size 524288 2019-12-07T19:33:58.714+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.714+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x33688~5c8 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.714+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x33688 0x33688~5c8 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.714+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.714+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x3688 2019-12-07T19:33:58.714+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x688 2019-12-07T19:33:58.714+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.714+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc50 and padding block with 0x3b0 2019-12-07T19:33:58.715+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x33c50 2019-12-07T19:33:58.715+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.715+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.715+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.715+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.715+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000493656s (0.000039998s flush + 0.000453658s kv commit) 2019-12-07T19:33:58.715+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56300> 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.715+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 kv_submitted 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56300 2019-12-07T19:33:58.715+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 finishing 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56300 onodes 0x558e83b09180 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac5f28 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f950 space 0x558e83ac5f28 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03198 in 0x558e80219880) _finish_write discard buffer(0x558e83c8fa70 space 0x558e83d03198 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03278 in 0x558e80219880) _finish_write discard buffer(0x558e83c8fb00 space 0x558e83d03278 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03358 in 0x558e80219880) _finish_write discard buffer(0x558e83c8fb90 space 0x558e83d03358 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03438 in 0x558e80219880) _finish_write discard buffer(0x558e83c8e750 space 0x558e83d03438 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03518 in 0x558e80219880) _finish_write discard buffer(0x558e81171050 space 0x558e83d03518 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d035f8 in 0x558e80219880) _finish_write discard buffer(0x558e83c8e3f0 space 0x558e83d035f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d036d8 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f290 space 0x558e83d036d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d037b8 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f170 space 0x558e83d037b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03898 in 0x558e80219880) _finish_write discard buffer(0x558e83c8e480 space 0x558e83d03898 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03978 in 0x558e80219880) _finish_write discard buffer(0x558e83c8ecf0 space 0x558e83d03978 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03a58 in 0x558e80219880) _finish_write discard buffer(0x558e83c8e990 space 0x558e83d03a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03b38 in 0x558e80219880) _finish_write discard buffer(0x558e83c8fc20 space 0x558e83d03b38 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03c18 in 0x558e80219880) _finish_write discard buffer(0x558e83c8eab0 space 0x558e83d03c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03cf8 in 0x558e80219880) _finish_write discard buffer(0x558e83d10090 space 0x558e83d03cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03dd8 in 0x558e80219880) _finish_write discard buffer(0x558e83d103f0 space 0x558e83d03dd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d37f28 in 0x558e80219880) _finish_write discard buffer(0x558e83c8f9e0 space 0x558e83d37f28 0x0~2000 writing nocache) 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56300 done 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.715+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56300 [] 2019-12-07T19:33:58.715+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.715+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.715+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=49, last_complete=21'7, committed=1, applied=1)) v2 -- 0x558e818ee600 con 0x558e818d7c00 2019-12-07T19:33:58.715+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.715+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.715+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 58 ==== MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=50, to_read={1:ee170468:::smithi13140796-25:head=0,4096,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e83c73900 con 0x558e818d7c00 2019-12-07T19:33:58.715+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81805ce0 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=50, to_read={1:ee170468:::smithi13140796-25:head=0,4096,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.715+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=50, to_read={1:ee170468:::smithi13140796-25:head=0,4096,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.715+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.715+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=50, to_read={1:ee170468:::smithi13140796-25:head=0,4096,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.715+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.715+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=50, to_read={1:ee170468:::smithi13140796-25:head=0,4096,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.715+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81805ce0 prio 127 cost 0 latency 0.000091 MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=50, to_read={1:ee170468:::smithi13140796-25:head=0,4096,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.715+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=50, to_read={1:ee170468:::smithi13140796-25:head=0,4096,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.715+0000 7fe2c8267700 25 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.715+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 0x0~1000 2019-12-07T19:33:58.715+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x19000 (102400) 2019-12-07T19:33:58.715+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.715+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f7a0 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac59d0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.715+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194f7a0 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac59d0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.715+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.715+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.716+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194f7a0 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac59d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.716+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83ac59e8 in 0x558e80219880) _discard 0x0~1000 2019-12-07T19:33:58.716+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83c8e5a0 space 0x558e83ac59e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.716+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 0x0~1000 = 4096 2019-12-07T19:33:58.716+0000 7fe2c8267700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.716+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] get_hash_info: Getting attr on 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.716+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] get_hash_info: not in cache 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.716+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.716+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] get_hash_info: found on disk, size 102400 2019-12-07T19:33:58.716+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# hinfo_key 2019-12-07T19:33:58.716+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# hinfo_key = 0 2019-12-07T19:33:58.716+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 21/18 ECSubReadReply(tid=50, attrs_read=0)) v2 -- 0x558e83b09680 con 0x558e818d7c00 2019-12-07T19:33:58.716+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81805ce0 finish 2019-12-07T19:33:58.716+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.717+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e83d69c00 0x558e83ae1080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.717+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e83d69c00 0x558e83ae1080 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.717+0000 7fe2e6d3a700 10 osd.0 21 ms_handle_authentication new session 0x558e81807ba0 con 0x558e83d69c00 entity osd.4 addr 2019-12-07T19:33:58.717+0000 7fe2e6d3a700 10 osd.0 21 ms_handle_authentication session 0x558e81807ba0 osd.4 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.717+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558e83d69c00 0x558e83ae1080 crc :-1 s=SESSION_ACCEPTING pgs=12 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x558e818fe400 2019-12-07T19:33:58.717+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558e83d69c00 0x558e83ae1080 crc :-1 s=SESSION_ACCEPTING pgs=12 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:58.717+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] conn(0x558e818fe400 0x558e818e5600 unknown :-1 s=READY pgs=12 cs=1 l=0 rx=0 tx=0).ready entity=osd.4 client_cookie=35acc3f1ec36344f server_cookie=b1c9bc281f9b9f1e in_seq=18 out_seq=17 2019-12-07T19:33:58.717+0000 7fe2d4a80700 2 osd.0 21 ms_handle_reset con 0x558e83d69c00 session 0x558e81807ba0 2019-12-07T19:33:58.717+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 19 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=17, reqid=client.4540.0:83, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e811b8800 con 0x558e818fe400 2019-12-07T19:33:58.717+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943080 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=17, reqid=client.4540.0:83, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.717+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=17, reqid=client.4540.0:83, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.717+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 20 ==== MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=18, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e83c72780 con 0x558e818fe400 2019-12-07T19:33:58.717+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943760 prio 127 cost 0 latency 0.000014 epoch 21 MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=18, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.718+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=18, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=17, reqid=client.4540.0:83, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=17, reqid=client.4540.0:83, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e81943080 prio 127 cost 0 latency 0.000093 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=17, reqid=client.4540.0:83, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=17, reqid=client.4540.0:83, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'4], crt=21'3) [] 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] rollforward: entry=21'4 (21'3) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:83 2019-12-07T19:33:58.690784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.718+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e811faf00 seq 12 2019-12-07T19:33:58.718+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811faf00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811faf00 onodes shared_blobs 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811faf00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 prepare 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811faf00 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 io_done 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e81943080 finish 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=18, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=18, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e81943760 prio 127 cost 0 latency 0.000203 MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=18, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=18, to_read={1:d07269f7:::smithi13140796-28:head=0,4096,0}, subchunks={1:d07269f7:::smithi13140796-28:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.718+0000 7fe2c6263700 25 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.718+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# 0x0~1000 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x1c000 (114688) 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d36bd0 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f880 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d36bd0 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f880 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.718+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.718+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.718+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.718+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.718+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.718+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.718+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.718+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x33c50~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.718+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x33c50 0x33c50~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.718+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.718+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x3c50 2019-12-07T19:33:58.718+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc50 2019-12-07T19:33:58.718+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.718+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc1 and padding block with 0xf3f 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d36bd0 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f880 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e8194f898 in 0x558e80219c00) _discard 0x0~1000 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e83c8e6c0 space 0x558e8194f898 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# 0x0~1000 = 4096 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: Getting attr on 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: not in cache 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] get_hash_info: found on disk, size 114688 2019-12-07T19:33:58.718+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# hinfo_key 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# hinfo_key = 0 2019-12-07T19:33:58.718+0000 7fe2c6263700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 21/18 ECSubReadReply(tid=18, attrs_read=0)) v2 -- 0x558e83c73680 con 0x558e818fe400 2019-12-07T19:33:58.718+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e81943760 finish 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.718+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x340c1 2019-12-07T19:33:58.718+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.718+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.718+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.718+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.718+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000311005s (0.000000102s flush + 0.000310903s kv commit) 2019-12-07T19:33:58.718+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.718+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.718+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811faf00> 2019-12-07T19:33:58.718+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.718+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 kv_submitted 2019-12-07T19:33:58.718+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811faf00 2019-12-07T19:33:58.718+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 finishing 2019-12-07T19:33:58.718+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811faf00 onodes 2019-12-07T19:33:58.718+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811faf00 done 2019-12-07T19:33:58.718+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:58.718+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811faf00 [] 2019-12-07T19:33:58.718+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.718+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.718+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=17, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558e818a8e00 con 0x558e818fe400 2019-12-07T19:33:58.718+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.718+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.723+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 10 ==== MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=11, reqid=client.4540.0:88, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83cbd800 con 0x558e818d6000 2019-12-07T19:33:58.723+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0a6e0 prio 127 cost 0 latency 0.000032 epoch 21 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=11, reqid=client.4540.0:88, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.723+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=11, reqid=client.4540.0:88, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.723+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 11 ==== MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:89, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e83cbe000 con 0x558e818d6000 2019-12-07T19:33:58.723+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e81943340 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:89, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.723+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:89, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=11, reqid=client.4540.0:88, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=11, reqid=client.4540.0:88, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e819e7000 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 osd.0 21 dequeue_op 0x558e83b0a6e0 prio 127 cost 0 latency 0.000122 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=11, reqid=client.4540.0:88, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.ds5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=11, reqid=client.4540.0:88, at_version=0'0, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=0'0 active mbc={}] append_log log((0'0,21'1], crt=0'0) [] 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 osd.0 pg_epoch: 21 pg[1.ds5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] rollforward: entry=21'1 (0'0) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:88 2019-12-07T19:33:58.696853+0000 0 ObjectCleanRegions clean_offsets: [491520~18446744073709060095], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log approx pg log length = 1 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.723+0000 7fe2c5261700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e819a1200 seq 6 2019-12-07T19:33:58.723+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes shared_blobs 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 osd.0 21 dequeue_op 0x558e83b0a6e0 finish 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:89, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:89, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e819e7000 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 osd.0 21 dequeue_op 0x558e81943340 prio 127 cost 0 latency 0.000239 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:89, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.ds5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x340c1~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:89, at_version=21'2, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x340c1 0x340c1~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x40c1 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc1 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x532 and padding block with 0xace 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'1], crt=21'1) [21'2 (21'1) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:89 2019-12-07T19:33:58.696891+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] add_log_entry 21'2 (21'1) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:89 2019-12-07T19:33:58.696891+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log approx pg log length = 2 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.723+0000 7fe2c5261700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e819a1500 seq 7 2019-12-07T19:33:58.723+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.ds5_head 5#1:bca18876:::smithi13140796-30:head# 3 keys 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.ds5_head 5#1:bca18876:::smithi13140796-30:head# 3 keys = 0 2019-12-07T19:33:58.723+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.ds5_head 5#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.ds5_head 5#1:bca18876:::smithi13140796-30:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.723+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1500 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1500 onodes 0x558e83cb2a00 shared_blobs 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore.extentmap(0x558e83cb2b08) update 5#1:bca18876:::smithi13140796-30:head# 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:bca18876:::smithi13140796-30:head# is 779 (485 bytes onode + 2 bytes spanning blobs + 292 bytes inline extents) 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 prepare 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1500 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 io_done 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'1 crt=21'1 lcod 21'1 active mbc={}] op_applied version 21'2 2019-12-07T19:33:58.723+0000 7fe2c5261700 10 osd.0 21 dequeue_op 0x558e81943340 finish 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x34532 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000209100s (0.000000143s flush + 0.000208957s kv commit) 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1819 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2a00 had 1 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1842 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1842 < min_flush_size 524288 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x34532~732 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.723+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x34532 0x34532~732 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x4532 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x532 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.723+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc64 and padding block with 0x39c 2019-12-07T19:33:58.723+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.723+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1200> 2019-12-07T19:33:58.723+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.723+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:33:58.723+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:33:58.723+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:33:58.723+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 2019-12-07T19:33:58.723+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:33:58.723+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1500 kv_submitted 2019-12-07T19:33:58.723+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:33:58.723+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.723+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.723+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=11, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e818a8800 con 0x558e818d6000 2019-12-07T19:33:58.723+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.723+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.724+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 59 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e811b9800 con 0x558e818d7c00 2019-12-07T19:33:58.724+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e818058c0 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.724+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.724+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.724+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.724+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.724+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.724+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e818058c0 prio 127 cost 0 latency 0.000072 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.724+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=56, reqid=client.4540.0:100, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.724+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'7], crt=21'5) [] 2019-12-07T19:33:58.724+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] rollforward: entry=21'6 (21'5) modify 1:ee170468:::smithi13140796-25:head by client.4540.0:74 2019-12-07T19:33:58.681620+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.724+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] rollforward: entry=21'7 (0'0) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:100 2019-12-07T19:33:58.711580+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.724+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.724+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.724+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.724+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.724+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.724+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e8195e900 seq 17 2019-12-07T19:33:58.724+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.724+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.724+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.724+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.724+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.724+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.724+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e900 onodes shared_blobs 2019-12-07T19:33:58.724+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.724+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 prepare 2019-12-07T19:33:58.724+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e900 2019-12-07T19:33:58.724+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 io_done 2019-12-07T19:33:58.724+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e818058c0 finish 2019-12-07T19:33:58.724+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.725+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 30 ==== MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=29, reqid=client.4540.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e80e53000 con 0x558e818d7000 2019-12-07T19:33:58.725+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81942160 prio 127 cost 0 latency 0.000034 epoch 21 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=29, reqid=client.4540.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.725+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=29, reqid=client.4540.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.725+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.725+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=29, reqid=client.4540.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.725+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.725+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=29, reqid=client.4540.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.725+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81942160 prio 127 cost 0 latency 0.000088 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=29, reqid=client.4540.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.fs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.725+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=29, reqid=client.4540.0:97, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.725+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'5], crt=21'4) [] 2019-12-07T19:33:58.725+0000 7fe2c8267700 20 osd.0 pg_epoch: 21 pg[1.fs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'5 (0'0) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:97 2019-12-07T19:33:58.707663+0000 0 ObjectCleanRegions clean_offsets: [540672~18446744073709010943], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.725+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.725+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.725+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.725+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.725+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:58.725+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e81a0b800 seq 14 2019-12-07T19:33:58.725+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:58.725+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:58.725+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.725+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.725+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:58.725+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b800 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.725+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b800 onodes shared_blobs 2019-12-07T19:33:58.725+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.725+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 prepare 2019-12-07T19:33:58.725+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b800 2019-12-07T19:33:58.725+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 io_done 2019-12-07T19:33:58.725+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e81942160 finish 2019-12-07T19:33:58.725+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.725+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 31 ==== MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:98, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e83ac8000 con 0x558e818d7000 2019-12-07T19:33:58.725+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a840 prio 127 cost 0 latency 0.000036 epoch 21 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:98, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.725+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:98, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:98, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:98, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83b0a840 prio 127 cost 0 latency 0.000096 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:98, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.fs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=30, reqid=client.4540.0:98, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'5], crt=21'5) [21'6 (21'5) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:98 2019-12-07T19:33:58.707684+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'5) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:98 2019-12-07T19:33:58.707684+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.726+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e8195fb00 seq 15 2019-12-07T19:33:58.726+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 3 keys 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 3 keys = 0 2019-12-07T19:33:58.726+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.726+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195fb00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195fb00 onodes 0x558e83b09b80 shared_blobs 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83b09c88) update 1#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:fc3a86a2:::smithi13140796-33:head# is 814 (485 bytes onode + 2 bytes spanning blobs + 327 bytes inline extents) 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195fb00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 prepare 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195fb00 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 io_done 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.726+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83b0a840 finish 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x34c64 2019-12-07T19:33:58.726+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.726+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.726+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.002690673s (0.000000080s flush + 0.002690593s kv commit) 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:33:58.726+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.726+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2228 < min_flush_size 524288 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 2 2019-12-07T19:33:58.726+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4082 < min_flush_size 524288 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b09b80 had 1 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:58.726+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4105 < min_flush_size 524288 2019-12-07T19:33:58.726+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4105 < min_flush_size 524288 2019-12-07T19:33:58.726+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.726+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x34c64~1009 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.726+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x34c64 0x34c64~1009 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x4c64 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc64 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc6d and padding block with 0x393 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1500> 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 kv_submitted 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1500 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 finishing 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1500 onodes 0x558e83cb2a00 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1500 done 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1500 [] 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.726+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=12, last_complete=21'2, committed=1, applied=1)) v2 -- 0x558e818a9400 con 0x558e818d6000 2019-12-07T19:33:58.726+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.726+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x35c6d 2019-12-07T19:33:58.726+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.726+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.726+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 3 cleaned 0 in 0.000176197s (0.000000139s flush + 0.000176058s kv commit) 2019-12-07T19:33:58.726+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e900,0x558e81a0b800,0x558e8195fb00> 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 kv_submitted 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e900 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 finishing 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e900 onodes 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e900 done 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e900 [] 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 kv_submitted 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b800 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 finishing 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b800 onodes 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 done 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 kv_submitted 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b800 [] 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 kv_submitted 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195fb00 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 finishing 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195fb00 onodes 0x558e83b09b80 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 done 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195fb00 [] 2019-12-07T19:33:58.726+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.726+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.726+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=56, last_complete=21'7, committed=1, applied=1)) v2 -- 0x558e81964a00 con 0x558e818d7c00 2019-12-07T19:33:58.726+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=29, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558e818a9800 con 0x558e818d7000 2019-12-07T19:33:58.726+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=30, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558e818a9600 con 0x558e818d7000 2019-12-07T19:33:58.726+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.726+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.726+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 60 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e83cbe800 con 0x558e818d7c00 2019-12-07T19:33:58.726+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0a840 prio 127 cost 0 latency 0.000030 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.726+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e83b0a840 prio 127 cost 0 latency 0.000319 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=57, reqid=client.4540.0:101, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'7], crt=21'7) [21'8 (21'7) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:101 2019-12-07T19:33:58.711613+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] add_log_entry 21'8 (21'7) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:101 2019-12-07T19:33:58.711613+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.727+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e81a0bb00 seq 18 2019-12-07T19:33:58.727+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# 3 keys 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# 3 keys = 0 2019-12-07T19:33:58.727+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.727+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0bb00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0bb00 onodes 0x558e83b09180 shared_blobs 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83b09288) update 3#1:e7f24557:::smithi13140796-34:head# 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:e7f24557:::smithi13140796-34:head# is 817 (485 bytes onode + 2 bytes spanning blobs + 330 bytes inline extents) 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0bb00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 prepare 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0bb00 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 io_done 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'7 crt=21'7 lcod 21'7 active mbc={}] op_applied version 21'8 2019-12-07T19:33:58.727+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e83b0a840 finish 2019-12-07T19:33:58.727+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.727+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.727+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.727+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1857 < min_flush_size 524288 2019-12-07T19:33:58.727+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b09180 had 1 2019-12-07T19:33:58.727+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.727+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1880 < min_flush_size 524288 2019-12-07T19:33:58.727+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1880 < min_flush_size 524288 2019-12-07T19:33:58.727+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.727+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x35c6d~758 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.727+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x35c6d 0x35c6d~758 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.727+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.727+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x5c6d 2019-12-07T19:33:58.727+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc6d 2019-12-07T19:33:58.727+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.727+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x3c5 and padding block with 0xc3b 2019-12-07T19:33:58.727+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 12 ==== MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:109, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 ==== 153524+0+0 (unknown 0 0 0) 0x558e811b9800 con 0x558e818d6000 2019-12-07T19:33:58.727+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e819434a0 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:109, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.727+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:109, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:109, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:109, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2) prio 127 cost 0 e21) pg 0x558e819e7000 2019-12-07T19:33:58.727+0000 7fe2c9269700 10 osd.0 21 dequeue_op 0x558e819434a0 prio 127 cost 0 latency 0.000175 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:109, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 pg pg[1.ds5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.727+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=14, reqid=client.4540.0:109, at_version=21'3, trim_to=0'0, roll_forward_to=21'1)) v2 2019-12-07T19:33:58.727+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'2], crt=21'1) [21'3 (0'0) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:109 2019-12-07T19:33:58.722797+0000 0 ObjectCleanRegions clean_offsets: [606208~18446744073708945407], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.727+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:109 2019-12-07T19:33:58.722797+0000 0 ObjectCleanRegions clean_offsets: [606208~18446744073708945407], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.727+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.727+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.727+0000 7fe2c9269700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.727+0000 7fe2c9269700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.727+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e83d4c000 seq 8 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.ds5_head 0x558e817b5860) get_onode oid 5#1:bf81b261:::smithi13140796-37:head# key 0x858000000000000001bf81b2'a!smithi13140796-37!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.727+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.ds5_head 5#1:bf81b261:::smithi13140796-37:head# 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1098 2019-12-07T19:33:58.727+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.ds5_head 5#1:bf81b261:::smithi13140796-37:head# = 0 2019-12-07T19:33:58.727+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.ds5_head 5#1:bf81b261:::smithi13140796-37:head# 2 keys 2019-12-07T19:33:58.727+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.ds5_head 5#1:bf81b261:::smithi13140796-37:head# 2 keys = 0 2019-12-07T19:33:58.727+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.ds5_head 5#1:bf81b261:::smithi13140796-37:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.727+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.ds5_head 5#1:bf81b261:::smithi13140796-37:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.727+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.ds5_head 5#1:bf81b261:::smithi13140796-37:head# 0x0~25000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#1:bf81b261:::smithi13140796-37:head# 0x0~25000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.727+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~25000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83d4c000 19 blobs 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.727+0000 7fe2c9269700 10 fbmap_alloc 0x558e80233300 allocate 0x25000/1000,25000,0 2019-12-07T19:33:58.727+0000 7fe2c9269700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x178000~25000/1000,25000,0 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x178000~25000] 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194e850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ff80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194e850 blob([0x178000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ff80 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e8194e850) get_ref 0x0~2000 Blob(0x558e8194e850 blob([0x178000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ff80 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e8194e850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e8194e850 blob([0x178000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ff80 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83d0ff98 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b28f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b28fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b28f50 blob([0x17a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b28fc0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b28f50) get_ref 0x0~2000 Blob(0x558e83b28f50 blob([0x17a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b28fc0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b28f50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83b28f50 blob([0x17a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28fc0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b28fd8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b290a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b29110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b290a0 blob([0x17c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29110 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b290a0) get_ref 0x0~2000 Blob(0x558e83b290a0 blob([0x17c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29110 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b290a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83b290a0 blob([0x17c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b29110 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b29128 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b29180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b291f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b29180 blob([0x17e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b291f0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29180) get_ref 0x0~2000 Blob(0x558e83b29180 blob([0x17e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b291f0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83b29180 blob([0x17e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b291f0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b29208 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b29260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b292d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b29260 blob([0x180000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b292d0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29260) get_ref 0x0~2000 Blob(0x558e83b29260 blob([0x180000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b292d0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83b29260 blob([0x180000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b292d0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b292e8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b29340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b293b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b29340 blob([0x182000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b293b0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29340) get_ref 0x0~2000 Blob(0x558e83b29340 blob([0x182000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b293b0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83b29340 blob([0x182000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b293b0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b293c8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b29420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b29490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b29420 blob([0x184000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29490 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29420) get_ref 0x0~2000 Blob(0x558e83b29420 blob([0x184000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29490 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83b29420 blob([0x184000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b29490 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b294a8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b29500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b29570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b29500 blob([0x186000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29570 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29500) get_ref 0x0~2000 Blob(0x558e83b29500 blob([0x186000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29570 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83b29500 blob([0x186000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b29570 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b29588 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b295e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b29650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b295e0 blob([0x188000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29650 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b295e0) get_ref 0x0~2000 Blob(0x558e83b295e0 blob([0x188000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29650 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b295e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83b295e0 blob([0x188000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b29650 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b29668 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b296c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b29730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b296c0 blob([0x18a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29730 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b296c0) get_ref 0x0~2000 Blob(0x558e83b296c0 blob([0x18a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29730 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b296c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83b296c0 blob([0x18a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b29730 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b29748 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b297a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b29810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b297a0 blob([0x18c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29810 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b297a0) get_ref 0x0~2000 Blob(0x558e83b297a0 blob([0x18c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29810 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b297a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83b297a0 blob([0x18c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b29810 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b29828 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b29880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b298f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b29880 blob([0x18e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b298f0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29880) get_ref 0x0~2000 Blob(0x558e83b29880 blob([0x18e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b298f0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83b29880 blob([0x18e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b298f0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b29908 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b29960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b299d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b29960 blob([0x190000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b299d0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29960) get_ref 0x0~2000 Blob(0x558e83b29960 blob([0x190000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b299d0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29960) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83b29960 blob([0x190000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b299d0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b299e8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b29a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b29ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b29a40 blob([0x192000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29ab0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29a40) get_ref 0x0~2000 Blob(0x558e83b29a40 blob([0x192000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29ab0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29a40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83b29a40 blob([0x192000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b29ab0 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b29ac8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b29b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b29b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b29b20 blob([0x194000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29b90 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29b20) get_ref 0x0~2000 Blob(0x558e83b29b20 blob([0x194000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29b90 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29b20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e83b29b20 blob([0x194000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b29b90 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b29ba8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b29c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b29c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b29c00 blob([0x196000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29c70 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29c00) get_ref 0x0~2000 Blob(0x558e83b29c00 blob([0x196000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29c70 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29c00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e83b29c00 blob([0x196000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b29c70 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b29c88 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b29ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b29d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b29ce0 blob([0x198000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29d50 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29ce0) get_ref 0x0~2000 Blob(0x558e83b29ce0 blob([0x198000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29d50 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29ce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e83b29ce0 blob([0x198000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b29d50 sbid 0x0)) 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b29d68 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.727+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b29dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b29e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b29dc0 blob([0x19a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29e30 sbid 0x0)) 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29dc0) get_ref 0x0~2000 Blob(0x558e83b29dc0 blob([0x19a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29e30 sbid 0x0)) 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29dc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e83b29dc0 blob([0x19a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b29e30 sbid 0x0)) 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b29e48 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b29ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b29f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b29ea0 blob([0x19c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29f10 sbid 0x0)) 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29ea0) get_ref 0x0~1000 Blob(0x558e83b29ea0 blob([0x19c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b29f10 sbid 0x0)) 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore.blob(0x558e83b29ea0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~1000: 0x0~1000 Blob(0x558e83b29ea0 blob([0x19c000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83b29f10 sbid 0x0)) 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83b29f28 in 0x558e80219dc0) _discard 0x0~1000 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x25000 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore.extentmap(0x558e83cb3008) dirty_range mark inline shard dirty 2019-12-07T19:33:58.728+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.ds5_head 5#1:bf81b261:::smithi13140796-37:head# 0x0~25000 = 0 2019-12-07T19:33:58.728+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.ds5_head 5#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.728+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.ds5_head 5#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.728+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.728+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:58.728+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c000 cost 233174 (20 ios * 4000 + 153174 bytes) 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c000 onodes 0x558e83cb2f00 shared_blobs 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore.extentmap(0x558e83cb3008) update 5#1:bf81b261:::smithi13140796-37:head# 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore.extentmap(0x558e83cb3008) update inline shard 361 bytes from 19 extents 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:bf81b261:::smithi13140796-37:head# is 780 (413 bytes onode + 2 bytes spanning blobs + 365 bytes inline extents) 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c000 allocated 0x[178000~25000] released 0x[] 2019-12-07T19:33:58.728+0000 7fe2c9269700 10 freelist allocate 0x178000~25000 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 freelist _xor first_key 0x100000 last_key 0x180000 2019-12-07T19:33:58.728+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 prepare 2019-12-07T19:33:58.728+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83d4c000 2019-12-07T19:33:58.728+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'2 crt=21'1 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.728+0000 7fe2c9269700 10 osd.0 21 dequeue_op 0x558e819434a0 finish 2019-12-07T19:33:58.728+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x363c5 2019-12-07T19:33:58.729+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.729+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.729+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.002177682s (0.000000078s flush + 0.002177604s kv commit) 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.729+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 aio_wait 2019-12-07T19:33:58.729+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c000 2019-12-07T19:33:58.729+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 io_done 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.729+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1521 < min_flush_size 524288 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2f00 had 1 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:33:58.729+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1544 < min_flush_size 524288 2019-12-07T19:33:58.729+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1544 < min_flush_size 524288 2019-12-07T19:33:58.729+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.729+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x363c5~608 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.729+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x363c5 0x363c5~608 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x63c5 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x3c5 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x9cd and padding block with 0x633 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x369cd 2019-12-07T19:33:58.729+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.729+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.729+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000218290s (0.000058333s flush + 0.000159957s kv commit) 2019-12-07T19:33:58.729+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0bb00,0x558e83d4c000> 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.730+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 kv_submitted 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0bb00 2019-12-07T19:33:58.730+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 finishing 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0bb00 onodes 0x558e83b09180 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0bb00 done 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.730+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0bb00 [] 2019-12-07T19:33:58.730+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.730+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 kv_submitted 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c000 2019-12-07T19:33:58.730+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 finishing 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c000 onodes 0x558e83cb2f00 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b28fd8 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8ea20 space 0x558e83b28fd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b29128 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d105a0 space 0x558e83b29128 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b29208 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10510 space 0x558e83b29208 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b292e8 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10360 space 0x558e83b292e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b293c8 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d106c0 space 0x558e83b293c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b294a8 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10750 space 0x558e83b294a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b29588 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d107e0 space 0x558e83b29588 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b29668 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10870 space 0x558e83b29668 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b29748 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10900 space 0x558e83b29748 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b29828 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10990 space 0x558e83b29828 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b29908 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10a20 space 0x558e83b29908 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b299e8 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10ab0 space 0x558e83b299e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b29ac8 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10b40 space 0x558e83b29ac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b29ba8 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10bd0 space 0x558e83b29ba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b29c88 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10c60 space 0x558e83b29c88 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b29d68 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10cf0 space 0x558e83b29d68 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b29e48 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10d80 space 0x558e83b29e48 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b29f28 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10e10 space 0x558e83b29f28 0x0~1000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0ff98 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10630 space 0x558e83d0ff98 0x0~2000 writing nocache) 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c000 done 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:33:58.730+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c000 [] 2019-12-07T19:33:58.730+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.730+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=57, last_complete=21'8, committed=1, applied=1)) v2 -- 0x558e818a8c00 con 0x558e818d7c00 2019-12-07T19:33:58.730+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.730+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.730+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=14, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e81955400 con 0x558e818d6000 2019-12-07T19:33:58.730+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.732+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 61 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e819d9800 con 0x558e818d7c00 2019-12-07T19:33:58.732+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81943b80 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.732+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.732+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 62 ==== MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=62, to_read={1:e7f24557:::smithi13140796-34:head=0,4096,0}, subchunks={1:e7f24557:::smithi13140796-34:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e83b09680 con 0x558e818d7c00 2019-12-07T19:33:58.732+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81942dc0 prio 127 cost 0 latency 0.000014 epoch 21 MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=62, to_read={1:e7f24557:::smithi13140796-34:head=0,4096,0}, subchunks={1:e7f24557:::smithi13140796-34:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.732+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=62, to_read={1:e7f24557:::smithi13140796-34:head=0,4096,0}, subchunks={1:e7f24557:::smithi13140796-34:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81943b80 prio 127 cost 0 latency 0.000111 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 pg pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=61, reqid=client.4540.0:101, at_version=0'0, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'8], crt=21'7) [] 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] rollforward: entry=21'8 (21'7) modify 1:e7f24557:::smithi13140796-34:head by client.4540.0:101 2019-12-07T19:33:58.711613+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.732+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e8195e300 seq 19 2019-12-07T19:33:58.732+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e300 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e300 onodes shared_blobs 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 prepare 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e300 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 io_done 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81943b80 finish 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=62, to_read={1:e7f24557:::smithi13140796-34:head=0,4096,0}, subchunks={1:e7f24557:::smithi13140796-34:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=62, to_read={1:e7f24557:::smithi13140796-34:head=0,4096,0}, subchunks={1:e7f24557:::smithi13140796-34:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81942dc0 prio 127 cost 0 latency 0.000223 MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=62, to_read={1:e7f24557:::smithi13140796-34:head=0,4096,0}, subchunks={1:e7f24557:::smithi13140796-34:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=62, to_read={1:e7f24557:::smithi13140796-34:head=0,4096,0}, subchunks={1:e7f24557:::smithi13140796-34:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.732+0000 7fe2c425f700 25 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.732+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# 0x0~1000 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x22000 (139264) 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65030 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5f10 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c65030 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5f10 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.732+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.732+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.732+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.732+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.732+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.732+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.732+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.732+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x369cd~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.732+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x369cd 0x369cd~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.732+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.732+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x69cd 2019-12-07T19:33:58.732+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x9cd 2019-12-07T19:33:58.732+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.732+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xe3e and padding block with 0x1c2 2019-12-07T19:33:58.732+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x36e3e 2019-12-07T19:33:58.732+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.732+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.732+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.732+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.732+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000144048s (0.000000087s flush + 0.000143961s kv commit) 2019-12-07T19:33:58.732+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c65030 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5f10 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5f28 in 0x558e80219880) _discard 0x0~1000 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e81170c60 space 0x558e83ac5f28 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# 0x0~1000 = 4096 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: Getting attr on 1:e7f24557:::smithi13140796-34:head 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: not in cache 1:e7f24557:::smithi13140796-34:head 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: found on disk, size 139264 2019-12-07T19:33:58.732+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# hinfo_key 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# hinfo_key = 0 2019-12-07T19:33:58.732+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 21/18 ECSubReadReply(tid=62, attrs_read=0)) v2 -- 0x558e83b08500 con 0x558e818d7c00 2019-12-07T19:33:58.732+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e81942dc0 finish 2019-12-07T19:33:58.732+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.732+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 63 ==== MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=58, reqid=client.4540.0:112, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 157620+0+0 (unknown 0 0 0) 0x558e819d8000 con 0x558e818d7c00 2019-12-07T19:33:58.732+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81942160 prio 127 cost 0 latency 0.000037 epoch 21 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=58, reqid=client.4540.0:112, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.732+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=58, reqid=client.4540.0:112, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.732+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.732+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=58, reqid=client.4540.0:112, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.732+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.732+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=58, reqid=client.4540.0:112, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558e81a98000 2019-12-07T19:33:58.732+0000 7fe2c8a68700 10 osd.0 21 dequeue_op 0x558e81942160 prio 127 cost 0 latency 0.000073 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=58, reqid=client.4540.0:112, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.es4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.732+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=58, reqid=client.4540.0:112, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.732+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (0'0) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:112 2019-12-07T19:33:58.727277+0000 0 ObjectCleanRegions clean_offsets: [622592~18446744073708929023], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.732+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (0'0) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:112 2019-12-07T19:33:58.727277+0000 0 ObjectCleanRegions clean_offsets: [622592~18446744073708929023], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.732+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.732+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.732+0000 7fe2c8a68700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.732+0000 7fe2c8a68700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e83d4c300 seq 9 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.es4_head 0x558e819c4960) get_onode oid 4#1:76d0c63f:::smithi13140796-38:head# key 0x84800000000000000176d0c6'?!smithi13140796-38!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.733+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1099 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# = 0 2019-12-07T19:33:58.733+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 2 keys 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 2 keys = 0 2019-12-07T19:33:58.733+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.733+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x0~26000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 4#1:76d0c63f:::smithi13140796-38:head# 0x0~26000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~26000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83d4c300 19 blobs 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 fbmap_alloc 0x558e80233300 allocate 0x26000/1000,26000,0 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x19d000~26000/1000,26000,0 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x19d000~26000] 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e88c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b28b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b28b60 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e817e88c0) get_ref 0x0~2000 Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b28b60 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e817e88c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28b60 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83b28b78 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c64a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c649a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c649a0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83c64a10) get_ref 0x0~2000 Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c649a0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83c64a10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c649a0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83c649b8 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c648c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81079810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81079810 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83c648c0) get_ref 0x0~2000 Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81079810 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83c648c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81079810 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e81079828 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194fb90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194fe30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194fe30 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e8194fb90) get_ref 0x0~2000 Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194fe30 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e8194fb90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fe30 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e8194fe48 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83da8930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da89a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da89a0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8930) get_ref 0x0~2000 Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da89a0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8930) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da89a0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83da89b8 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83da8a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da8a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da8a80 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8a10) get_ref 0x0~2000 Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da8a80 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8a10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8a80 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83da8a98 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83da8af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da8b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da8b60 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8af0) get_ref 0x0~2000 Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da8b60 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8af0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8b60 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83da8b78 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83da8bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da8c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da8c40 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8bd0) get_ref 0x0~2000 Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da8c40 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8bd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8c40 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83da8c58 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b29030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ff10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ff10 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83b29030) get_ref 0x0~2000 Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0ff10 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83b29030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ff10 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83d0ff28 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83da8cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da8d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da8d20 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8cb0) get_ref 0x0~2000 Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da8d20 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8cb0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8d20 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83da8d38 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83da8d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da8e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da8e00 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8d90) get_ref 0x0~2000 Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da8e00 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8d90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8e00 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83da8e18 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83da8e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da8ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da8ee0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8e70) get_ref 0x0~2000 Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da8ee0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8e70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8ee0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83da8ef8 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83da8f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da8fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da8fc0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8f50) get_ref 0x0~2000 Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da8fc0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da8f50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8fc0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83da8fd8 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83da9030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da90a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da90a0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da9030) get_ref 0x0~2000 Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da90a0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da9030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da90a0 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83da90b8 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83da9110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da9180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da9180 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da9110) get_ref 0x0~2000 Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da9180 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da9110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9180 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83da9198 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83da91f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da9260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da9260 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da91f0) get_ref 0x0~2000 Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da9260 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da91f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9260 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83da9278 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83da92d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da9340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da9340 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da92d0) get_ref 0x0~2000 Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da9340 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da92d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9340 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83da9358 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83da93b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da9420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da9420 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da93b0) get_ref 0x0~2000 Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da9420 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da93b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9420 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83da9438 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83da9490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da9500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da9500 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da9490) get_ref 0x0~2000 Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da9500 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.blob(0x558e83da9490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9500 sbid 0x0)) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e83da9518 in 0x558e80219ea0) _discard 0x0~2000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x26000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.extentmap(0x558e811dd008) dirty_range mark inline shard dirty 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x0~26000 = 0 2019-12-07T19:33:58.733+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.733+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c300 cost 237270 (20 ios * 4000 + 157270 bytes) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c300 onodes 0x558e811dcf00 shared_blobs 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.extentmap(0x558e811dd008) update 4#1:76d0c63f:::smithi13140796-38:head# 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore.extentmap(0x558e811dd008) update inline shard 364 bytes from 19 extents 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 4#1:76d0c63f:::smithi13140796-38:head# is 783 (413 bytes onode + 2 bytes spanning blobs + 368 bytes inline extents) 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c300 allocated 0x[19d000~26000] released 0x[] 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 freelist allocate 0x19d000~26000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 prepare 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83d4c300 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.733+0000 7fe2c8a68700 10 osd.0 21 dequeue_op 0x558e81942160 finish 2019-12-07T19:33:58.733+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.733+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 aio_wait 2019-12-07T19:33:58.733+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c300 2019-12-07T19:33:58.733+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 io_done 2019-12-07T19:33:58.733+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.733+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.733+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.733+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1495 < min_flush_size 524288 2019-12-07T19:33:58.733+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dcf00 had 1 2019-12-07T19:33:58.733+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:33:58.733+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1518 < min_flush_size 524288 2019-12-07T19:33:58.733+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1518 < min_flush_size 524288 2019-12-07T19:33:58.733+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.733+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x36e3e~5ee to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.733+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x36e3e 0x36e3e~5ee to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.733+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.733+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x6e3e 2019-12-07T19:33:58.733+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xe3e 2019-12-07T19:33:58.733+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.733+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x42c and padding block with 0xbd4 2019-12-07T19:33:58.734+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3742c 2019-12-07T19:33:58.734+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.734+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.734+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.734+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.734+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000143108s (0.000048042s flush + 0.000095066s kv commit) 2019-12-07T19:33:58.734+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.734+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 32 ==== MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:98, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e811b8800 con 0x558e818d7000 2019-12-07T19:33:58.734+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a420 prio 127 cost 0 latency 0.000071 epoch 21 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:98, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.734+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:98, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.734+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.734+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:98, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.734+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.734+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:98, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.734+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e83b0a420 prio 127 cost 0 latency 0.000124 MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:98, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.734+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.fs1 21/18 ECSubWrite(tid=33, reqid=client.4540.0:98, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.734+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'6], crt=21'5) [] 2019-12-07T19:33:58.734+0000 7fe2c8267700 20 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'6 (21'5) modify 1:fc3a86a2:::smithi13140796-33:head by client.4540.0:98 2019-12-07T19:33:58.707684+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.734+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.734+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.734+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.734+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.734+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:33:58.734+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e81a0a600 seq 16 2019-12-07T19:33:58.734+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:33:58.734+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:33:58.734+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.734+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.734+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:33:58.734+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.734+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a600 onodes shared_blobs 2019-12-07T19:33:58.734+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.734+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 prepare 2019-12-07T19:33:58.734+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a600 2019-12-07T19:33:58.734+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 io_done 2019-12-07T19:33:58.734+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e83b0a420 finish 2019-12-07T19:33:58.734+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.734+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.734+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.734+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.734+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:33:58.734+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.734+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.734+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.734+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3742c~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.734+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3742c 0x3742c~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.734+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.734+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x742c 2019-12-07T19:33:58.734+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x42c 2019-12-07T19:33:58.734+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.734+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x89d and padding block with 0x763 2019-12-07T19:33:58.734+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3789d 2019-12-07T19:33:58.734+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.734+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.734+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.735+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.735+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000188587s (0.000000093s flush + 0.000188494s kv commit) 2019-12-07T19:33:58.735+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.735+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 33 ==== MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=34, to_read={1:fc3a86a2:::smithi13140796-33:head=0,4096,0}, subchunks={1:fc3a86a2:::smithi13140796-33:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e83c73680 con 0x558e818d7000 2019-12-07T19:33:58.735+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e819431e0 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=34, to_read={1:fc3a86a2:::smithi13140796-33:head=0,4096,0}, subchunks={1:fc3a86a2:::smithi13140796-33:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.735+0000 7fe2e6d3a700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=34, to_read={1:fc3a86a2:::smithi13140796-33:head=0,4096,0}, subchunks={1:fc3a86a2:::smithi13140796-33:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.735+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.735+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=34, to_read={1:fc3a86a2:::smithi13140796-33:head=0,4096,0}, subchunks={1:fc3a86a2:::smithi13140796-33:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.735+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.735+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=34, to_read={1:fc3a86a2:::smithi13140796-33:head=0,4096,0}, subchunks={1:fc3a86a2:::smithi13140796-33:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e81a9b000 2019-12-07T19:33:58.735+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e819431e0 prio 127 cost 0 latency 0.000073 MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=34, to_read={1:fc3a86a2:::smithi13140796-33:head=0,4096,0}, subchunks={1:fc3a86a2:::smithi13140796-33:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.735+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpRead(1.fs1 21/18 ECSubRead(tid=34, to_read={1:fc3a86a2:::smithi13140796-33:head=0,4096,0}, subchunks={1:fc3a86a2:::smithi13140796-33:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.735+0000 7fe2c425f700 25 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.735+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x0~1000 2019-12-07T19:33:58.735+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x21000 (135168) 2019-12-07T19:33:58.735+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.735+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f960 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.735+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f960 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.735+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.735+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.736+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 13 ==== MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:109, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83cbf800 con 0x558e818d6000 2019-12-07T19:33:58.736+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0a000 prio 127 cost 0 latency 0.000036 epoch 21 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:109, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.736+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:109, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.736+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.736+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:109, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.736+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.736+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:109, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558e819e7000 2019-12-07T19:33:58.736+0000 7fe2c5261700 10 osd.0 21 dequeue_op 0x558e83b0a000 prio 127 cost 0 latency 0.000082 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:109, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.ds5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] 2019-12-07T19:33:58.736+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=18, reqid=client.4540.0:109, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.736+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'1 active mbc={}] append_log log((0'0,21'3], crt=21'1) [] 2019-12-07T19:33:58.736+0000 7fe2c5261700 20 osd.0 pg_epoch: 21 pg[1.ds5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'2 (21'1) modify 1:bca18876:::smithi13140796-30:head by client.4540.0:89 2019-12-07T19:33:58.696891+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.736+0000 7fe2c5261700 20 osd.0 pg_epoch: 21 pg[1.ds5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:109 2019-12-07T19:33:58.722797+0000 0 ObjectCleanRegions clean_offsets: [606208~18446744073708945407], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.736+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.736+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.736+0000 7fe2c5261700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.736+0000 7fe2c5261700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.736+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:58.736+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e811fbb00 seq 9 2019-12-07T19:33:58.736+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:58.736+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.736+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.736+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.736+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:58.736+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fbb00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.736+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fbb00 onodes shared_blobs 2019-12-07T19:33:58.736+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fbb00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.736+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 prepare 2019-12-07T19:33:58.736+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fbb00 2019-12-07T19:33:58.736+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 io_done 2019-12-07T19:33:58.736+0000 7fe2c5261700 10 osd.0 21 dequeue_op 0x558e83b0a000 finish 2019-12-07T19:33:58.736+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.736+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:33:58.736+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.736+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.736+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.736+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3789d~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.736+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3789d 0x3789d~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.736+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.736+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x789d 2019-12-07T19:33:58.736+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x89d 2019-12-07T19:33:58.736+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.736+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd0e and padding block with 0x2f2 2019-12-07T19:33:58.736+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x37d0e 2019-12-07T19:33:58.736+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.736+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.736+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.736+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000187258s (0.000000157s flush + 0.000187101s kv commit) 2019-12-07T19:33:58.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.736+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f960 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.736+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e8194f978 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:33:58.736+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e83d10120 space 0x558e8194f978 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.736+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x0~1000 = 4096 2019-12-07T19:33:58.736+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.736+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: Getting attr on 1:fc3a86a2:::smithi13140796-33:head 2019-12-07T19:33:58.736+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: not in cache 1:fc3a86a2:::smithi13140796-33:head 2019-12-07T19:33:58.736+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:33:58.736+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: found on disk, size 135168 2019-12-07T19:33:58.736+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# hinfo_key 2019-12-07T19:33:58.736+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# hinfo_key = 0 2019-12-07T19:33:58.736+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.fs0 21/18 ECSubReadReply(tid=34, attrs_read=0)) v2 -- 0x558e83b09680 con 0x558e818d7000 2019-12-07T19:33:58.736+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e819431e0 finish 2019-12-07T19:33:58.736+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e300,0x558e83d4c300,0x558e81a0a600,0x558e811fbb00> 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 kv_submitted 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e300 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 finishing 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e300 onodes 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 done 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e300 [] 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 kv_submitted 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c300 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 finishing 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c300 onodes 0x558e811dcf00 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81079828 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8e1b0 space 0x558e81079828 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194fe48 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8f950 space 0x558e8194fe48 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b28b78 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8e090 space 0x558e83b28b78 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c649b8 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8e120 space 0x558e83c649b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0ff28 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8e750 space 0x558e83d0ff28 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da89b8 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8fa70 space 0x558e83da89b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da8a98 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8fb00 space 0x558e83da8a98 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da8b78 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8fb90 space 0x558e83da8b78 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da8c58 in 0x558e80219ea0) _finish_write discard buffer(0x558e81171050 space 0x558e83da8c58 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da8d38 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8e3f0 space 0x558e83da8d38 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da8e18 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8f290 space 0x558e83da8e18 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da8ef8 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8f170 space 0x558e83da8ef8 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da8fd8 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8e480 space 0x558e83da8fd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da90b8 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8ecf0 space 0x558e83da90b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da9198 in 0x558e80219ea0) _finish_write discard buffer(0x558e83d11050 space 0x558e83da9198 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da9278 in 0x558e80219ea0) _finish_write discard buffer(0x558e83d10fc0 space 0x558e83da9278 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da9358 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8e990 space 0x558e83da9358 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da9438 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8fc20 space 0x558e83da9438 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da9518 in 0x558e80219ea0) _finish_write discard buffer(0x558e83c8eab0 space 0x558e83da9518 0x0~2000 writing nocache) 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c300 done 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c300 [] 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 kv_submitted 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a600 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 finishing 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a600 onodes 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a600 done 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a600 [] 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 kv_submitted 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fbb00 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 finishing 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fbb00 onodes 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fbb00 done 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fbb00 [] 2019-12-07T19:33:58.737+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.737+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.737+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=18, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e818a9000 con 0x558e818d6000 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.737+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.737+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=61, last_complete=21'8, committed=1, applied=1)) v2 -- 0x558e818eee00 con 0x558e818d7c00 2019-12-07T19:33:58.737+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 21/18 ECSubWriteReply(tid=33, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558e818a8200 con 0x558e818d7000 2019-12-07T19:33:58.737+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.737+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.737+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 14 ==== MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:110, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e83cbd000 con 0x558e818d6000 2019-12-07T19:33:58.737+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0a6e0 prio 127 cost 0 latency 0.000034 epoch 21 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:110, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.737+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:110, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:110, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:110, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558e819e7000 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 osd.0 21 dequeue_op 0x558e83b0a6e0 prio 127 cost 0 latency 0.000076 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:110, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.ds5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=21, reqid=client.4540.0:110, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'3], crt=21'3) [21'4 (21'3) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:110 2019-12-07T19:33:58.722827+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] add_log_entry 21'4 (21'3) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:110 2019-12-07T19:33:58.722827+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.737+0000 7fe2c9269700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e83d4cf00 seq 10 2019-12-07T19:33:58.737+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.ds5_head 5#1:bf81b261:::smithi13140796-37:head# 3 keys 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.ds5_head 5#1:bf81b261:::smithi13140796-37:head# 3 keys = 0 2019-12-07T19:33:58.737+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.ds5_head 5#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.ds5_head 5#1:bf81b261:::smithi13140796-37:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.737+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4cf00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4cf00 onodes 0x558e83cb2f00 shared_blobs 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 bluestore.extentmap(0x558e83cb3008) update 5#1:bf81b261:::smithi13140796-37:head# 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:bf81b261:::smithi13140796-37:head# is 852 (485 bytes onode + 2 bytes spanning blobs + 365 bytes inline extents) 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4cf00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 prepare 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4cf00 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 io_done 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.737+0000 7fe2c9269700 10 osd.0 21 dequeue_op 0x558e83b0a6e0 finish 2019-12-07T19:33:58.737+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.737+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.737+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.737+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 754 < min_flush_size 524288 2019-12-07T19:33:58.737+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1903 < min_flush_size 524288 2019-12-07T19:33:58.737+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2f00 had 1 2019-12-07T19:33:58.737+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:33:58.737+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1926 < min_flush_size 524288 2019-12-07T19:33:58.737+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1926 < min_flush_size 524288 2019-12-07T19:33:58.737+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.737+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x37d0e~786 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.737+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x37d0e 0x37d0e~786 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.737+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.737+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x7d0e 2019-12-07T19:33:58.737+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd0e 2019-12-07T19:33:58.737+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.737+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x494 and padding block with 0xb6c 2019-12-07T19:33:58.738+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x38494 2019-12-07T19:33:58.738+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.738+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.738+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.738+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.738+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000216125s (0.000000080s flush + 0.000216045s kv commit) 2019-12-07T19:33:58.738+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.738+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.738+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4cf00> 2019-12-07T19:33:58.738+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.738+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 kv_submitted 2019-12-07T19:33:58.738+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4cf00 2019-12-07T19:33:58.738+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 finishing 2019-12-07T19:33:58.738+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4cf00 onodes 0x558e83cb2f00 2019-12-07T19:33:58.738+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4cf00 done 2019-12-07T19:33:58.738+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:33:58.738+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4cf00 [] 2019-12-07T19:33:58.738+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.738+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.738+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.738+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=21, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558e80264c00 con 0x558e818d6000 2019-12-07T19:33:58.738+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.739+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83d68000 0x558e83ae1b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.739+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83d68000 0x558e83ae1b80 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.740+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 15 ==== MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=22, reqid=client.4540.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83cbd800 con 0x558e818d6000 2019-12-07T19:33:58.740+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0a160 prio 127 cost 0 latency 0.000033 epoch 21 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=22, reqid=client.4540.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.740+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=22, reqid=client.4540.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.740+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.740+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=22, reqid=client.4540.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.740+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.740+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=22, reqid=client.4540.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558e819e7000 2019-12-07T19:33:58.740+0000 7fe2c5261700 10 osd.0 21 dequeue_op 0x558e83b0a160 prio 127 cost 0 latency 0.000078 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=22, reqid=client.4540.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.ds5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.740+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=22, reqid=client.4540.0:110, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.740+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'4], crt=21'3) [] 2019-12-07T19:33:58.740+0000 7fe2c5261700 20 osd.0 pg_epoch: 21 pg[1.ds5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] rollforward: entry=21'4 (21'3) modify 1:bf81b261:::smithi13140796-37:head by client.4540.0:110 2019-12-07T19:33:58.722827+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.740+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.740+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.740+0000 7fe2c5261700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.740+0000 7fe2c5261700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.740+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:58.740+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e811fa600 seq 11 2019-12-07T19:33:58.740+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:58.740+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.740+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.740+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.740+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:58.740+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.740+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa600 onodes shared_blobs 2019-12-07T19:33:58.740+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.740+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 prepare 2019-12-07T19:33:58.740+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa600 2019-12-07T19:33:58.740+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 io_done 2019-12-07T19:33:58.740+0000 7fe2c5261700 10 osd.0 21 dequeue_op 0x558e83b0a160 finish 2019-12-07T19:33:58.740+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.740+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.740+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.740+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.740+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:33:58.740+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.740+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.740+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.740+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x38494~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.740+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x38494 0x38494~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.740+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.740+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x8494 2019-12-07T19:33:58.740+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x494 2019-12-07T19:33:58.740+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.740+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x905 and padding block with 0x6fb 2019-12-07T19:33:58.741+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x38905 2019-12-07T19:33:58.741+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.741+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.741+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.741+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.741+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000124067s (0.000000074s flush + 0.000123993s kv commit) 2019-12-07T19:33:58.741+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.741+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.741+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa600> 2019-12-07T19:33:58.741+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.741+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 kv_submitted 2019-12-07T19:33:58.741+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa600 2019-12-07T19:33:58.741+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 finishing 2019-12-07T19:33:58.741+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa600 onodes 2019-12-07T19:33:58.741+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa600 done 2019-12-07T19:33:58.741+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:33:58.741+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa600 [] 2019-12-07T19:33:58.741+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.741+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.741+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=22, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558e81955400 con 0x558e818d6000 2019-12-07T19:33:58.741+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.741+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.741+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 171531 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.742+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.742+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=58, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e811f7200 con 0x558e818d7c00 2019-12-07T19:33:58.742+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.2 172.21.15.131:0/37077 6 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:58.742768+0000/22.214487565s send_stamp 22.214487565s delta_ub 0.017257144s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8a00 con 0x558e8182c000 2019-12-07T19:33:58.742+0000 7fe2e6d3a700 20 osd.0 21 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016882366s,0.017257144s]) 2019-12-07T19:33:58.742+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:58.742768+0000/22.214487565s send_stamp 22.197605199s delta_ub 18446744073.692669250s) v5 -- 0x558e81239c00 con 0x558e8182c000 2019-12-07T19:33:58.742+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.742+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.2 172.21.15.131:0/37077 6 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:58.742768+0000/22.214487565s send_stamp 22.214487565s delta_ub 0.017257144s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e811f7200 con 0x558e8182d400 2019-12-07T19:33:58.742+0000 7fe2e7d3c700 20 osd.0 21 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016125076s,0.017257144s]) 2019-12-07T19:33:58.742+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:58.742768+0000/22.214487565s send_stamp 22.198362489s delta_ub 18446744073.693426540s) v5 -- 0x558e818a8c00 con 0x558e8182d400 2019-12-07T19:33:58.743+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 64 ==== MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=65, reqid=client.4540.0:112, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e819db800 con 0x558e818d7c00 2019-12-07T19:33:58.743+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81943340 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=65, reqid=client.4540.0:112, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.743+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=65, reqid=client.4540.0:112, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.743+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.743+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=65, reqid=client.4540.0:112, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.743+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.743+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=65, reqid=client.4540.0:112, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558e81a98000 2019-12-07T19:33:58.743+0000 7fe2c4a60700 10 osd.0 21 dequeue_op 0x558e81943340 prio 127 cost 0 latency 0.000056 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=65, reqid=client.4540.0:112, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.es4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.743+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=65, reqid=client.4540.0:112, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.743+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [] 2019-12-07T19:33:58.743+0000 7fe2c4a60700 20 osd.0 pg_epoch: 21 pg[1.es4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (0'0) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:112 2019-12-07T19:33:58.727277+0000 0 ObjectCleanRegions clean_offsets: [622592~18446744073708929023], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.743+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.743+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.743+0000 7fe2c4a60700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.743+0000 7fe2c4a60700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.743+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:33:58.743+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e819a0000 seq 10 2019-12-07T19:33:58.743+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:58.743+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.743+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.743+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.743+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:58.743+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0000 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.743+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0000 onodes shared_blobs 2019-12-07T19:33:58.743+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.743+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 prepare 2019-12-07T19:33:58.743+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0000 2019-12-07T19:33:58.743+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 io_done 2019-12-07T19:33:58.743+0000 7fe2c4a60700 10 osd.0 21 dequeue_op 0x558e81943340 finish 2019-12-07T19:33:58.743+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.743+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 65 ==== MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=66, reqid=client.4540.0:113, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e80e53000 con 0x558e818d7c00 2019-12-07T19:33:58.743+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e818051e0 prio 127 cost 0 latency 0.000023 epoch 21 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=66, reqid=client.4540.0:113, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.743+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=66, reqid=client.4540.0:113, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.743+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.743+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=66, reqid=client.4540.0:113, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.743+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.743+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=66, reqid=client.4540.0:113, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558e81a98000 2019-12-07T19:33:58.743+0000 7fe2c8a68700 10 osd.0 21 dequeue_op 0x558e818051e0 prio 127 cost 0 latency 0.000126 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=66, reqid=client.4540.0:113, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.es4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.743+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=66, reqid=client.4540.0:113, at_version=21'4, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.743+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'3 (0'0,21'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'3], crt=21'3) [21'4 (21'3) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:113 2019-12-07T19:33:58.727310+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.743+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] add_log_entry 21'4 (21'3) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:113 2019-12-07T19:33:58.727310+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.743+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.743+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.743+0000 7fe2c8a68700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.743+0000 7fe2c8a68700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.743+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:33:58.743+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e83d4c900 seq 11 2019-12-07T19:33:58.743+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 3 keys 2019-12-07T19:33:58.743+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 3 keys = 0 2019-12-07T19:33:58.743+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.743+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.744+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:58.744+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.744+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.744+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.744+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.744+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:58.744+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c900 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.744+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c900 onodes 0x558e811dcf00 shared_blobs 2019-12-07T19:33:58.744+0000 7fe2c8a68700 20 bluestore.extentmap(0x558e811dd008) update 4#1:76d0c63f:::smithi13140796-38:head# 2019-12-07T19:33:58.744+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 4#1:76d0c63f:::smithi13140796-38:head# is 855 (485 bytes onode + 2 bytes spanning blobs + 368 bytes inline extents) 2019-12-07T19:33:58.744+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.744+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 prepare 2019-12-07T19:33:58.744+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c900 2019-12-07T19:33:58.744+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 io_done 2019-12-07T19:33:58.744+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active mbc={}] op_applied version 21'4 2019-12-07T19:33:58.744+0000 7fe2c8a68700 10 osd.0 21 dequeue_op 0x558e818051e0 finish 2019-12-07T19:33:58.744+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.744+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.744+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.744+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.744+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 2 2019-12-07T19:33:58.744+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3009 < min_flush_size 524288 2019-12-07T19:33:58.744+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dcf00 had 1 2019-12-07T19:33:58.744+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:33:58.744+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3032 < min_flush_size 524288 2019-12-07T19:33:58.744+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3032 < min_flush_size 524288 2019-12-07T19:33:58.744+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.744+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x38905~bd8 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.744+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x38905 0x38905~bd8 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.744+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.744+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x8905 2019-12-07T19:33:58.744+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x905 2019-12-07T19:33:58.744+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.744+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x4dd and padding block with 0xb23 2019-12-07T19:33:58.744+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x394dd 2019-12-07T19:33:58.744+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.744+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.744+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.744+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.744+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000108236s (0.000000086s flush + 0.000108150s kv commit) 2019-12-07T19:33:58.744+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a0000,0x558e83d4c900> 2019-12-07T19:33:58.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.744+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 kv_submitted 2019-12-07T19:33:58.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0000 2019-12-07T19:33:58.744+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 finishing 2019-12-07T19:33:58.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0000 onodes 2019-12-07T19:33:58.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0000 done 2019-12-07T19:33:58.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c900 kv_submitted 2019-12-07T19:33:58.744+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0000 [] 2019-12-07T19:33:58.744+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.744+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 kv_submitted 2019-12-07T19:33:58.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c900 2019-12-07T19:33:58.744+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 finishing 2019-12-07T19:33:58.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c900 onodes 0x558e811dcf00 2019-12-07T19:33:58.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c900 done 2019-12-07T19:33:58.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:33:58.744+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c900 [] 2019-12-07T19:33:58.744+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.744+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.744+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=65, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e81964a00 con 0x558e818d7c00 2019-12-07T19:33:58.744+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=66, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558e811f6600 con 0x558e818d7c00 2019-12-07T19:33:58.744+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.745+0000 7fe2e6d3a700 10 osd.0 21 ms_handle_authentication new session 0x558e818e9860 con 0x558e83d68000 entity client.0 addr 2019-12-07T19:33:58.745+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 66 ==== MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=67, reqid=client.4540.0:113, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83acb800 con 0x558e818d7c00 2019-12-07T19:33:58.745+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81805ce0 prio 127 cost 0 latency 0.000040 epoch 21 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=67, reqid=client.4540.0:113, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.745+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=67, reqid=client.4540.0:113, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.745+0000 7fe2e6d3a700 10 osd.0 21 ms_handle_authentication session 0x558e818e9860 client.0 has caps osdcap[grant(rwx)] 'allow rwx' 2019-12-07T19:33:58.745+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.745+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=67, reqid=client.4540.0:113, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.745+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.745+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=67, reqid=client.4540.0:113, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558e81a98000 2019-12-07T19:33:58.745+0000 7fe2c4a60700 10 osd.0 21 dequeue_op 0x558e81805ce0 prio 127 cost 0 latency 0.000098 MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=67, reqid=client.4540.0:113, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] 2019-12-07T19:33:58.745+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] _handle_message: MOSDECSubOpWrite(1.es4 21/18 ECSubWrite(tid=67, reqid=client.4540.0:113, at_version=0'0, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.745+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log log((0'0,21'4], crt=21'3) [] 2019-12-07T19:33:58.745+0000 7fe2c4a60700 20 osd.0 pg_epoch: 21 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] rollforward: entry=21'4 (21'3) modify 1:76d0c63f:::smithi13140796-38:head by client.4540.0:113 2019-12-07T19:33:58.727310+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.745+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log approx pg log length = 4 2019-12-07T19:33:58.745+0000 7fe2c4a60700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.745+0000 7fe2c4a60700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.745+0000 7fe2c4a60700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.745+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:33:58.745+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e81a0b500 seq 12 2019-12-07T19:33:58.745+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:33:58.745+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:33:58.745+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.745+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.745+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:33:58.745+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.745+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b500 onodes shared_blobs 2019-12-07T19:33:58.745+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.745+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 prepare 2019-12-07T19:33:58.745+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b500 2019-12-07T19:33:58.745+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 io_done 2019-12-07T19:33:58.745+0000 7fe2c4a60700 10 osd.0 21 dequeue_op 0x558e81805ce0 finish 2019-12-07T19:33:58.745+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.745+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.745+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.745+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.745+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:33:58.745+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.745+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.745+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.745+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x394dd~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.745+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x394dd 0x394dd~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.745+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.745+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x94dd 2019-12-07T19:33:58.745+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x4dd 2019-12-07T19:33:58.745+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.745+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x94e and padding block with 0x6b2 2019-12-07T19:33:58.745+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3994e 2019-12-07T19:33:58.745+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.745+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.745+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.745+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.745+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000289050s (0.000000088s flush + 0.000288962s kv commit) 2019-12-07T19:33:58.745+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.745+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.745+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0b500> 2019-12-07T19:33:58.745+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.745+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 kv_submitted 2019-12-07T19:33:58.745+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b500 2019-12-07T19:33:58.745+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 finishing 2019-12-07T19:33:58.745+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b500 onodes 2019-12-07T19:33:58.745+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b500 done 2019-12-07T19:33:58.745+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:33:58.745+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b500 [] 2019-12-07T19:33:58.745+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.745+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.745+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.es0 21/18 ECSubWriteReply(tid=67, last_complete=21'4, committed=1, applied=1)) v2 -- 0x558e811f7200 con 0x558e818d7c00 2019-12-07T19:33:58.745+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.745+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:58.748+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/224485236 conn(0x558e83d68000 0x558e83ae1b80 crc :-1 s=READY pgs=14 cs=0 l=1 rx=0 tx=0).ready entity=client.4540 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:33:58.748+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 1 ==== osd_op(client.4540.0:115 1.2s0 1.5e84fb82 (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 264+0+638976 (crc 0 0 0) 0x558e83da6000 con 0x558e83d68000 2019-12-07T19:33:58.748+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943760 prio 63 cost 638976 latency 0.000027 epoch 21 osd_op(client.4540.0:115 1.2s0 1.5e84fb82 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.748+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:115 1.2s0 1.5e84fb82 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 638976 e21) 2019-12-07T19:33:58.748+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 2 ==== osd_op(client.4540.0:116 1.2s0 1.5e84fb82 (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 226+0+63 (crc 0 0 0) 0x558e83da62c0 con 0x558e83d68000 2019-12-07T19:33:58.748+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81942b00 prio 63 cost 63 latency 0.000011 epoch 21 osd_op(client.4540.0:116 1.2s0 1.5e84fb82 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.748+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:116 1.2s0 1.5e84fb82 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) 2019-12-07T19:33:58.748+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 3 ==== osd_op(client.4540.0:117 1.2s0 1.5e84fb82 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 ==== 226+0+0 (crc 0 0 0) 0x558e83da6580 con 0x558e83d68000 2019-12-07T19:33:58.748+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e819438c0 prio 63 cost 0 latency 0.000009 epoch 21 osd_op(client.4540.0:117 1.2s0 1.5e84fb82 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.748+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:117 1.2s0 1.5e84fb82 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.748+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.748+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:115 1.2s0 1.5e84fb82 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 638976 e21) queued 2019-12-07T19:33:58.748+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.748+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:115 1.2s0 1.5e84fb82 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 638976 e21) pg 0x558e81075000 2019-12-07T19:33:58.748+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81943760 prio 63 cost 638976 latency 0.000107 osd_op(client.4540.0:115 1.2s0 1.5e84fb82 (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.748+0000 7fe2caa6c700 10 osd.0 21 maybe_share_map con 0x558e83d68000 172.21.15.131:0/224485236 map epoch 0 -> 21 (as per caller) 2019-12-07T19:33:58.748+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:115 1.2s0 1.5e84fb82 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.748+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:115 1.2s0 1:41df217a:::smithi13140796-39:head [append 0~638976 in=638976b,stat] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.748+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.748+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:115 1.2s0 1:41df217a:::smithi13140796-39:head [append 0~638976 in=638976b,stat] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write may_read -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.748+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_object_context: obc NOT found in cache: 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.748+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# _ 2019-12-07T19:33:58.748+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:41df217a:::smithi13140796-39:head# key 0x80800000000000000141df21'z!smithi13140796-39!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# _ = -2 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_object_context: no obc for soid 1:41df217a:::smithi13140796-39:head but can_create 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] create_object_context 0x558e81222900 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] populate_obc_watchers 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_object_context: 0x558e81222900 1:41df217a:::smithi13140796-39:head rwstate(none n=0 w=0) oi: 1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) ssc: 0x558e811237a0 snapset: 0=[]:{} 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] find_object_context 1:41df217a:::smithi13140796-39:head @head oi=1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.749+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_op oi 1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_op obc obc(1:41df217a:::smithi13140796-39:head(dne) rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] execute_ctx 0x558e81228e00 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] execute_ctx 1:41df217a:::smithi13140796-39:head [append 0~638976 in=638976b,stat] ov 0'0 av 21'1 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op 1:41df217a:::smithi13140796-39:head [append 0~638976 in=638976b,stat] 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op append 0~638976 in=638976b 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op 1:41df217a:::smithi13140796-39:head [write 0~638976 in=638976b] 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op write 0~638976 in=638976b 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] clean_regions modifiedclean_offsets: [638976~18446744073708912639], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] do_osd_op stat 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] stat oi has 638976 0.000000 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] make_writeable 1:41df217a:::smithi13140796-39:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] setting DIRTY flag 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] make_writeable 1:41df217a:::smithi13140796-39:head done, snapset=0=[]:{} 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] finish_ctx 1:41df217a:::smithi13140796-39:head 0x558e81228e00 op modify 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] set mtime to 2019-12-07T19:33:58.731888+0000 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] final snapset 0=[]:{} in 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] finish_ctx object 1:41df217a:::smithi13140796-39:head marks clean_regions clean_offsets: [638976~18446744073708912639], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] execute_ctx alloc reply 0x558e83c72c80 result 0 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] calc_trim_to_aggressive limit = 0'0 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] op order client.4540 tid 115 (first) 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] new_repop rep_tid 0 on osd_op(client.4540.0:115 1.2s0 1:41df217a:::smithi13140796-39:head [append 0~638976 in=638976b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] new_repop: repgather(0x558e811238c0 0'0 rep_tid=0 committed?=0 r=0) 2019-12-07T19:33:58.749+0000 7fe2caa6c700 7 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] issue_repop rep_tid 0 o 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] submit_transaction: op Op(1:41df217a:::smithi13140796-39:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:115 client_op=osd_op(client.4540.0:115 1.2s0 1:41df217a:::smithi13140796-39:head [append 0~638976 in=638976b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_hash_info: Getting attr on 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] get_hash_info: not in cache 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:41df217a:::smithi13140796-39:head# key 0x80800000000000000141df21'z!smithi13140796-39!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): 1:41df217a:::smithi13140796-39:head projected size 638976 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] start_rmw: Op(1:41df217a:::smithi13140796-39:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:115 client_op=osd_op(client.4540.0:115 1.2s0 1:41df217a:::smithi13140796-39:head [append 0~638976 in=638976b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:head=[0~638976]}) 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_state_to_reads: Op(1:41df217a:::smithi13140796-39:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:115 client_op=osd_op(client.4540.0:115 1.2s0 1:41df217a:::smithi13140796-39:head [append 0~638976 in=638976b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:head=[0~638976]}) 2019-12-07T19:33:58.749+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: starting commit on Op(1:41df217a:::smithi13140796-39:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:115 client_op=osd_op(client.4540.0:115 1.2s0 1:41df217a:::smithi13140796-39:head [append 0~638976 in=638976b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:head=[0~638976]}) 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:41df217a:::smithi13140796-39:head)[ Extent(0~638976:1) ) 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] generate_transactions: 1:41df217a:::smithi13140796-39:head, current size is 0 buffers are {0~638976(638976)} 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): new_size start 0 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): adding buffer_update 0,638976 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): to_append: {0~638976(638976)} 2019-12-07T19:33:58.749+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): appending 0~638976 2019-12-07T19:33:58.750+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 4 ==== osd_op(client.4540.0:127 1.2s0 1.84ed6192 (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 264+0+720896 (crc 0 0 0) 0x558e83da6840 con 0x558e83d68000 2019-12-07T19:33:58.750+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943ce0 prio 63 cost 720896 latency 0.000021 epoch 21 osd_op(client.4540.0:127 1.2s0 1.84ed6192 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.750+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:127 1.2s0 1.84ed6192 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 720896 e21) 2019-12-07T19:33:58.750+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 5 ==== osd_op(client.4540.0:128 1.2s0 1.84ed6192 (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 226+0+63 (crc 0 0 0) 0x558e83da6b00 con 0x558e83d68000 2019-12-07T19:33:58.750+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a6e0 prio 63 cost 63 latency 0.000008 epoch 21 osd_op(client.4540.0:128 1.2s0 1.84ed6192 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.750+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:128 1.2s0 1.84ed6192 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) 2019-12-07T19:33:58.750+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 6 ==== osd_op(client.4540.0:129 1.2s0 1.84ed6192 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 ==== 226+0+0 (crc 0 0 0) 0x558e83da6dc0 con 0x558e83d68000 2019-12-07T19:33:58.750+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a2c0 prio 63 cost 0 latency 0.000007 epoch 21 osd_op(client.4540.0:129 1.2s0 1.84ed6192 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.750+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:129 1.2s0 1.84ed6192 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.751+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] encode_and_write: 1:41df217a:::smithi13140796-39:head new_size 638976 2019-12-07T19:33:58.751+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): 1:41df217a:::smithi13140796-39:head resetting hinfo to logical size 638976 2019-12-07T19:33:58.751+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] operator(): marking append 0 2019-12-07T19:33:58.751+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:41df217a:::smithi13140796-39:head)[ Extent(0~638976:1) ) 2019-12-07T19:33:58.751+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: written: {1:41df217a:::smithi13140796-39:head={0~638976(638976)}} 2019-12-07T19:33:58.751+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: op: Op(1:41df217a:::smithi13140796-39:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:115 client_op=osd_op(client.4540.0:115 1.2s0 1:41df217a:::smithi13140796-39:head [append 0~638976 in=638976b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:head=[0~638976]}) 2019-12-07T19:33:58.751+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: written_set: {1:41df217a:::smithi13140796-39:head=[0~638976]} 2019-12-07T19:33:58.751+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] try_reads_to_commit: 1:41df217a:::smithi13140796-39:head,{0~638976(638976)} 2019-12-07T19:33:58.751+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 21/18 ECSubWrite(tid=0, reqid=client.4540.0:115, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e83aca800 con 0x558e818d7c00 2019-12-07T19:33:58.751+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 21/18 ECSubWrite(tid=0, reqid=client.4540.0:115, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e819da800 con 0x558e818d7000 2019-12-07T19:33:58.751+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 21/18 ECSubWrite(tid=0, reqid=client.4540.0:115, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e819da000 con 0x558e818fe400 2019-12-07T19:33:58.751+0000 7fe2caa6c700 10 osd.0 21 send_incremental_map 18 -> 21 to 0x558e818d6800 v2:172.21.15.109:6802/36936 2019-12-07T19:33:58.751+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:cd1b972f:::inc_osdmap.20:0# 0x0~0 2019-12-07T19:33:58.751+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:33:58.751+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:58.751+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81867810 blob([0x31000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558e81867ce0 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:33:58.751+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:cd1b972f:::inc_osdmap.20:0# 0x0~24d = 589 2019-12-07T19:33:58.751+0000 7fe2caa6c700 10 osd.0 21 add_map_inc_bl 20 589 bytes 2019-12-07T19:33:58.751+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c21b972f:::inc_osdmap.21:0# 0x0~0 2019-12-07T19:33:58.751+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:33:58.751+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194fea0 blob([0x34000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558e8194ff10 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c21b972f:::inc_osdmap.21:0# 0x0~24d = 589 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 21 add_map_inc_bl 21 589 bytes 2019-12-07T19:33:58.752+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(19..21 src has 1..21) v4 -- 0x558e83dd0480 con 0x558e818d6800 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 21 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 18 -> 21 (shared) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s4 21/18 ECSubWrite(tid=0, reqid=client.4540.0:115, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e819d9000 con 0x558e818d6800 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 21 send_incremental_map 19 -> 21 to 0x558e818ffc00 v2:172.21.15.109:6818/36937 2019-12-07T19:33:58.752+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- osd_map(20..21 src has 1..21) v4 -- 0x558e83dd06c0 con 0x558e818ffc00 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 21 maybe_share_map con 0x558e818ffc00 v2:172.21.15.109:6818/36937 map epoch 19 -> 21 (shared) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=0, reqid=client.4540.0:115, at_version=21'1, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e819db000 con 0x558e818ffc00 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 earliest_dup_version = 0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 trim 21'1 (0'0) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:115 2019-12-07T19:33:58.731888+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( empty local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=0'0 mlcod 0'0 active+clean] append_log log((0'0,0'0], crt=0'0) [21'1 (0'0) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:115 2019-12-07T19:33:58.731888+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] add_log_entry 21'1 (0'0) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:115 2019-12-07T19:33:58.731888+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log approx pg log length = 1 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'1, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e83c57500 seq 6 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:41df217a:::smithi13140796-39:head# key 0x80800000000000000141df21'z!smithi13140796-39!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.752+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1100 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# = 0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2 keys 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2 keys = 0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 0x0~27000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 0#1:41df217a:::smithi13140796-39:head# 0x0~27000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~27000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83c57500 20 blobs 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 fbmap_alloc 0x558e80233300 allocate 0x27000/1000,27000,0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x1c3000~27000/1000,27000,0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x1c3000~27000] 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd2ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd2ee0 blob([0x1c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2e70 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd2ee0) get_ref 0x0~2000 Blob(0x558e83dd2ee0 blob([0x1c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2e70 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd2ee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83dd2ee0 blob([0x1c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e70 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd2e88 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd2bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd2bd0 blob([0x1c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2e00 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd2bd0) get_ref 0x0~2000 Blob(0x558e83dd2bd0 blob([0x1c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2e00 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd2bd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83dd2bd0 blob([0x1c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e00 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd2e18 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd30a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd30a0 blob([0x1c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3180 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd30a0) get_ref 0x0~2000 Blob(0x558e83dd30a0 blob([0x1c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3180 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd30a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83dd30a0 blob([0x1c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3180 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3198 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd2fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd2fc0 blob([0x1c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2c40 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd2fc0) get_ref 0x0~2000 Blob(0x558e83dd2fc0 blob([0x1c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2c40 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd2fc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83dd2fc0 blob([0x1c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2c40 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd2c58 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd3030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd3030 blob([0x1cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2b60 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3030) get_ref 0x0~2000 Blob(0x558e83dd3030 blob([0x1cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2b60 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83dd3030 blob([0x1cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2b60 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd2b78 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd3110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd3110 blob([0x1cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2af0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3110) get_ref 0x0~2000 Blob(0x558e83dd3110 blob([0x1cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2af0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83dd3110 blob([0x1cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2af0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd2b08 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd23f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd23f0 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2380 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd23f0) get_ref 0x0~2000 Blob(0x558e83dd23f0 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2380 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd23f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83dd23f0 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2380 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd2398 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd2310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd22a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd2310 blob([0x1d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd22a0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd2310) get_ref 0x0~2000 Blob(0x558e83dd2310 blob([0x1d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd22a0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd2310) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83dd2310 blob([0x1d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd22a0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd22b8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd2a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd29a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd2a80 blob([0x1d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd29a0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd2a80) get_ref 0x0~2000 Blob(0x558e83dd2a80 blob([0x1d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd29a0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd2a80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83dd2a80 blob([0x1d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd29a0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd29b8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd3260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd32d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd3260 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd32d0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3260) get_ref 0x0~2000 Blob(0x558e83dd3260 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd32d0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83dd3260 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd32d0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd32e8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd3340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd33b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd3340 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd33b0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3340) get_ref 0x0~2000 Blob(0x558e83dd3340 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd33b0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83dd3340 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd33b0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd33c8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd3420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd3420 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3490 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3420) get_ref 0x0~2000 Blob(0x558e83dd3420 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3490 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83dd3420 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3490 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd34a8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd3500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd3500 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3570 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3500) get_ref 0x0~2000 Blob(0x558e83dd3500 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3570 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83dd3500 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3570 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3588 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd35e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd35e0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3650 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd35e0) get_ref 0x0~2000 Blob(0x558e83dd35e0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3650 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd35e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83dd35e0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3650 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3668 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd36c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd36c0 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3730 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd36c0) get_ref 0x0~2000 Blob(0x558e83dd36c0 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3730 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd36c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e83dd36c0 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3730 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3748 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd37a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd37a0 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3810 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd37a0) get_ref 0x0~2000 Blob(0x558e83dd37a0 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3810 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd37a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e83dd37a0 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3810 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3828 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd3880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd38f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd3880 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd38f0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3880) get_ref 0x0~2000 Blob(0x558e83dd3880 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd38f0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e83dd3880 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd38f0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3908 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd3960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd39d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd3960 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd39d0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3960) get_ref 0x0~2000 Blob(0x558e83dd3960 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd39d0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3960) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e83dd3960 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd39d0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd39e8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd3a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd3a40 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3ab0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3a40) get_ref 0x0~2000 Blob(0x558e83dd3a40 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3ab0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3a40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e83dd3a40 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3ab0 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3ac8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd3b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd3b20 blob([0x1e9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3b90 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3b20) get_ref 0x0~1000 Blob(0x558e83dd3b20 blob([0x1e9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3b90 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd3b20) get_ref init 0x1000, 1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~1000: 0x0~1000 Blob(0x558e83dd3b20 blob([0x1e9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83dd3b90 sbid 0x0)) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3ba8 in 0x558e802195e0) _discard 0x0~1000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x27000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e83cb3a08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 0x0~27000 = 0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c57500 cost 322174 (40 ios * 4000 + 162174 bytes) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c57500 onodes 0x558e83cb3900 shared_blobs 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e83cb3a08) update 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e83cb3a08) update inline shard 380 bytes from 20 extents 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:41df217a:::smithi13140796-39:head# is 799 (413 bytes onode + 2 bytes spanning blobs + 384 bytes inline extents) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c57500 allocated 0x[1c3000~27000] released 0x[] 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 freelist allocate 0x1c3000~27000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 prepare 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83c57500 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 lua=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_applied version 21'1 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] eval_repop repgather(0x558e811238c0 21'1 rep_tid=0 committed?=0 r=0) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81943760 finish 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:116 1.2s0 1.5e84fb82 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) queued 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:116 1.2s0 1.5e84fb82 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) pg 0x558e81075000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81942b00 prio 63 cost 63 latency 0.004042 osd_op(client.4540.0:116 1.2s0 1.5e84fb82 (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:116 1.2s0 1.5e84fb82 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x558e81222900 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x558e81222900 1:41df217a:::smithi13140796-39:head rwstate(excl n=1 w=0) oi: 1:41df217a:::smithi13140796-39:head(21'1 client.4540.0:115 dirty|data_digest s 638976 uv 1 dd b038798b alloc_hint [0 0 0]) exists: 1 ssc: 0x558e811237a0 snapset: 0=[]:{} 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:41df217a:::smithi13140796-39:head @head oi=1:41df217a:::smithi13140796-39:head(21'1 client.4540.0:115 dirty|data_digest s 638976 uv 1 dd b038798b alloc_hint [0 0 0]) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:41df217a:::smithi13140796-39:head(21'1 client.4540.0:115 dirty|data_digest s 638976 uv 1 dd b038798b alloc_hint [0 0 0]) 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81942b00 finish 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:117 1.2s0 1.5e84fb82 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:117 1.2s0 1.5e84fb82 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e819438c0 prio 63 cost 0 latency 0.004128 osd_op(client.4540.0:117 1.2s0 1.5e84fb82 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.752+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:117 1.2s0 1.5e84fb82 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.752+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x558e81222900 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x558e81222900 1:41df217a:::smithi13140796-39:head rwstate(excl n=1 w=1) oi: 1:41df217a:::smithi13140796-39:head(21'1 client.4540.0:115 dirty|data_digest s 638976 uv 1 dd b038798b alloc_hint [0 0 0]) exists: 1 ssc: 0x558e811237a0 snapset: 0=[]:{} 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:41df217a:::smithi13140796-39:head @head oi=1:41df217a:::smithi13140796-39:head(21'1 client.4540.0:115 dirty|data_digest s 638976 uv 1 dd b038798b alloc_hint [0 0 0]) 2019-12-07T19:33:58.753+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:41df217a:::smithi13140796-39:head(21'1 client.4540.0:115 dirty|data_digest s 638976 uv 1 dd b038798b alloc_hint [0 0 0]) 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e819438c0 finish 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:127 1.2s0 1.84ed6192 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 720896 e21) queued 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:127 1.2s0 1.84ed6192 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 720896 e21) pg 0x558e81075000 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81943ce0 prio 63 cost 720896 latency 0.002784 osd_op(client.4540.0:127 1.2s0 1.84ed6192 (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:127 1.2s0 1.84ed6192 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:127 1.2s0 1:4986b721:::smithi13140796-43:head [append 0~720896 in=720896b,stat] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:127 1.2s0 1:4986b721:::smithi13140796-43:head [append 0~720896 in=720896b,stat] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write may_read -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: obc NOT found in cache: 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.753+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# _ 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:4986b721:::smithi13140796-43:head# key 0x8080000000000000014986b7'!!smithi13140796-43!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# _ = -2 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: no obc for soid 1:4986b721:::smithi13140796-43:head but can_create 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] create_object_context 0x558e83dd0900 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] populate_obc_watchers 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x558e83dd0900 1:4986b721:::smithi13140796-43:head rwstate(none n=0 w=0) oi: 1:4986b721:::smithi13140796-43:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) ssc: 0x558e811239e0 snapset: 0=[]:{} 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:4986b721:::smithi13140796-43:head @head oi=1:4986b721:::smithi13140796-43:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.753+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:4986b721:::smithi13140796-43:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op obc obc(1:4986b721:::smithi13140796-43:head(dne) rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 0x558e80ff3800 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 1:4986b721:::smithi13140796-43:head [append 0~720896 in=720896b,stat] ov 0'0 av 21'2 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op 1:4986b721:::smithi13140796-43:head [append 0~720896 in=720896b,stat] 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op append 0~720896 in=720896b 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op 1:4986b721:::smithi13140796-43:head [write 0~720896 in=720896b] 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op write 0~720896 in=720896b 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] clean_regions modifiedclean_offsets: [720896~18446744073708830719], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op stat 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] stat oi has 720896 0.000000 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] make_writeable 1:4986b721:::smithi13140796-43:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] setting DIRTY flag 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] make_writeable 1:4986b721:::smithi13140796-43:head done, snapset=0=[]:{} 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] finish_ctx 1:4986b721:::smithi13140796-43:head 0x558e80ff3800 op modify 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] set mtime to 2019-12-07T19:33:58.750442+0000 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] final snapset 0=[]:{} in 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] finish_ctx object 1:4986b721:::smithi13140796-43:head marks clean_regions clean_offsets: [720896~18446744073708830719], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx alloc reply 0x558e81a03900 result 0 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] calc_trim_to_aggressive limit = 0'0 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op order client.4540 tid 127 (first) 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] new_repop rep_tid 1 on osd_op(client.4540.0:127 1.2s0 1:4986b721:::smithi13140796-43:head [append 0~720896 in=720896b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] new_repop: repgather(0x558e81123b00 0'0 rep_tid=1 committed?=0 r=0) 2019-12-07T19:33:58.753+0000 7fe2caa6c700 7 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] issue_repop rep_tid 1 o 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] submit_transaction: op Op(1:4986b721:::smithi13140796-43:head v=21'2 tt=0'0 tid=1 reqid=client.4540.0:127 client_op=osd_op(client.4540.0:127 1.2s0 1:4986b721:::smithi13140796-43:head [append 0~720896 in=720896b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: Getting attr on 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: not in cache 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:4986b721:::smithi13140796-43:head# key 0x8080000000000000014986b7'!!smithi13140796-43!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): 1:4986b721:::smithi13140796-43:head projected size 720896 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] start_rmw: Op(1:4986b721:::smithi13140796-43:head v=21'2 tt=0'0 tid=1 reqid=client.4540.0:127 client_op=osd_op(client.4540.0:127 1.2s0 1:4986b721:::smithi13140796-43:head [append 0~720896 in=720896b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4986b721:::smithi13140796-43:head=[0~720896]}) 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_state_to_reads: Op(1:4986b721:::smithi13140796-43:head v=21'2 tt=0'0 tid=1 reqid=client.4540.0:127 client_op=osd_op(client.4540.0:127 1.2s0 1:4986b721:::smithi13140796-43:head [append 0~720896 in=720896b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4986b721:::smithi13140796-43:head=[0~720896]}) 2019-12-07T19:33:58.753+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: starting commit on Op(1:4986b721:::smithi13140796-43:head v=21'2 tt=0'0 tid=1 reqid=client.4540.0:127 client_op=osd_op(client.4540.0:127 1.2s0 1:4986b721:::smithi13140796-43:head [append 0~720896 in=720896b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4986b721:::smithi13140796-43:head=[0~720896]}) 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:41df217a:::smithi13140796-39:head)[ Extent(0~638976:1) Extents(1:4986b721:::smithi13140796-43:head)[ Extent(0~720896:2) ) 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] generate_transactions: 1:4986b721:::smithi13140796-43:head, current size is 0 buffers are {0~720896(720896)} 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): new_size start 0 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): adding buffer_update 0,720896 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): to_append: {0~720896(720896)} 2019-12-07T19:33:58.753+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): appending 0~720896 2019-12-07T19:33:58.754+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 21 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e80264c00 con 0x558e818fe400 2019-12-07T19:33:58.754+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a580 prio 196 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.754+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.754+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 6 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818ef400 con 0x558e818ffc00 2019-12-07T19:33:58.754+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e818058c0 prio 196 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.754+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.755+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] encode_and_write: 1:4986b721:::smithi13140796-43:head new_size 720896 2019-12-07T19:33:58.755+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): 1:4986b721:::smithi13140796-43:head resetting hinfo to logical size 720896 2019-12-07T19:33:58.755+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): marking append 0 2019-12-07T19:33:58.755+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:41df217a:::smithi13140796-39:head)[ Extent(0~638976:1) Extents(1:4986b721:::smithi13140796-43:head)[ Extent(0~720896:2) ) 2019-12-07T19:33:58.755+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: written: {1:4986b721:::smithi13140796-43:head={0~720896(720896)}} 2019-12-07T19:33:58.755+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: op: Op(1:4986b721:::smithi13140796-43:head v=21'2 tt=0'0 tid=1 reqid=client.4540.0:127 client_op=osd_op(client.4540.0:127 1.2s0 1:4986b721:::smithi13140796-43:head [append 0~720896 in=720896b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4986b721:::smithi13140796-43:head=[0~720896]}) 2019-12-07T19:33:58.755+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: written_set: {1:4986b721:::smithi13140796-43:head=[0~720896]} 2019-12-07T19:33:58.755+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: 1:4986b721:::smithi13140796-43:head,{0~720896(720896)} 2019-12-07T19:33:58.755+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 21/18 ECSubWrite(tid=1, reqid=client.4540.0:127, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e83acb800 con 0x558e818d7c00 2019-12-07T19:33:58.755+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 21/18 ECSubWrite(tid=1, reqid=client.4540.0:127, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e80e53000 con 0x558e818d7000 2019-12-07T19:33:58.755+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 21/18 ECSubWrite(tid=1, reqid=client.4540.0:127, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e83cbf800 con 0x558e818fe400 2019-12-07T19:33:58.755+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s4 21/18 ECSubWrite(tid=1, reqid=client.4540.0:127, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e83cbd000 con 0x558e818d6800 2019-12-07T19:33:58.755+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=1, reqid=client.4540.0:127, at_version=21'2, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e811b9800 con 0x558e818ffc00 2019-12-07T19:33:58.755+0000 7fe2caa6c700 20 earliest_dup_version = 0 2019-12-07T19:33:58.755+0000 7fe2caa6c700 20 trim 21'2 (0'0) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:127 2019-12-07T19:33:58.750442+0000 0 ObjectCleanRegions clean_offsets: [720896~18446744073708830719], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.755+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'1 (0'0,21'1] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log log((0'0,21'1], crt=0'0) [21'2 (0'0) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:127 2019-12-07T19:33:58.750442+0000 0 ObjectCleanRegions clean_offsets: [720896~18446744073708830719], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.755+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] add_log_entry 21'2 (0'0) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:127 2019-12-07T19:33:58.750442+0000 0 ObjectCleanRegions clean_offsets: [720896~18446744073708830719], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.755+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log approx pg log length = 2 2019-12-07T19:33:58.755+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.755+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.755+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'2, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.756+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e83c57800 seq 7 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:4986b721:::smithi13140796-43:head# key 0x8080000000000000014986b7'!!smithi13140796-43!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.756+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1101 2019-12-07T19:33:58.756+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# = 0 2019-12-07T19:33:58.756+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 2 keys 2019-12-07T19:33:58.756+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 2 keys = 0 2019-12-07T19:33:58.756+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.756+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.756+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 0x0~2c000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 0#1:4986b721:::smithi13140796-43:head# 0x0~2c000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.756+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~2c000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83c57800 22 blobs 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.756+0000 7fe2caa6c700 10 fbmap_alloc 0x558e80233300 allocate 0x2c000/1000,2c000,0 2019-12-07T19:33:58.756+0000 7fe2caa6c700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x1ea000~2c000/1000,2c000,0 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x1ea000~2c000] 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff2850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff27e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff2850 blob([0x1ea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff27e0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff2850) get_ref 0x0~2000 Blob(0x558e83ff2850 blob([0x1ea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff27e0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff2850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83ff2850 blob([0x1ea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff27e0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff27f8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff28c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff28c0 blob([0x1ec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2930 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff28c0) get_ref 0x0~2000 Blob(0x558e83ff28c0 blob([0x1ec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2930 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff28c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83ff28c0 blob([0x1ec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2930 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff2948 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff2e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff2e00 blob([0x1ee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2cb0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff2e00) get_ref 0x0~2000 Blob(0x558e83ff2e00 blob([0x1ee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2cb0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff2e00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83ff2e00 blob([0x1ee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2cb0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff2cc8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff2e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff2e70 blob([0x1f0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2d20 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff2e70) get_ref 0x0~2000 Blob(0x558e83ff2e70 blob([0x1f0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2d20 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff2e70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83ff2e70 blob([0x1f0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2d20 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff2d38 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff2ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff2ee0 blob([0x1f2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2d90 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff2ee0) get_ref 0x0~2000 Blob(0x558e83ff2ee0 blob([0x1f2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2d90 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff2ee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83ff2ee0 blob([0x1f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2d90 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff2da8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff30a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff30a0 blob([0x1f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2f50 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff30a0) get_ref 0x0~2000 Blob(0x558e83ff30a0 blob([0x1f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2f50 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff30a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83ff30a0 blob([0x1f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2f50 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff2f68 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff3110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff3110 blob([0x1f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2fc0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3110) get_ref 0x0~2000 Blob(0x558e83ff3110 blob([0x1f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff2fc0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83ff3110 blob([0x1f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2fc0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff2fd8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff3180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff3180 blob([0x1f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3030 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3180) get_ref 0x0~2000 Blob(0x558e83ff3180 blob([0x1f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3030 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83ff3180 blob([0x1f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3030 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3048 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff31f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff31f0 blob([0x1fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3260 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff31f0) get_ref 0x0~2000 Blob(0x558e83ff31f0 blob([0x1fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3260 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff31f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83ff31f0 blob([0x1fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3260 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3278 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff32d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff32d0 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3340 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff32d0) get_ref 0x0~2000 Blob(0x558e83ff32d0 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3340 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff32d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83ff32d0 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3340 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3358 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff33b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff33b0 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3420 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff33b0) get_ref 0x0~2000 Blob(0x558e83ff33b0 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3420 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff33b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83ff33b0 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3420 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3438 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff3490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff3490 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3500 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3490) get_ref 0x0~2000 Blob(0x558e83ff3490 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3500 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83ff3490 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3500 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3518 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff3570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff35e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff3570 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff35e0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3570) get_ref 0x0~2000 Blob(0x558e83ff3570 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff35e0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83ff3570 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff35e0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff35f8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff3650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff36c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff3650 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff36c0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3650) get_ref 0x0~2000 Blob(0x558e83ff3650 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff36c0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3650) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83ff3650 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff36c0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff36d8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff3730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff37a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff3730 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff37a0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3730) get_ref 0x0~2000 Blob(0x558e83ff3730 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff37a0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e83ff3730 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff37a0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff37b8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff3810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff3810 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3880 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3810) get_ref 0x0~2000 Blob(0x558e83ff3810 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3880 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e83ff3810 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3880 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3898 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff38f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff38f0 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3960 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff38f0) get_ref 0x0~2000 Blob(0x558e83ff38f0 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3960 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff38f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e83ff38f0 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3960 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3978 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff39d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff39d0 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3a40 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff39d0) get_ref 0x0~2000 Blob(0x558e83ff39d0 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3a40 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff39d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e83ff39d0 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3a40 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3a58 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff3ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff3ab0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3b20 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3ab0) get_ref 0x0~2000 Blob(0x558e83ff3ab0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3b20 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3ab0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e83ff3ab0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3b20 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3b38 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff3b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff3b90 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3c00 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3b90) get_ref 0x0~2000 Blob(0x558e83ff3b90 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3c00 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3b90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e83ff3b90 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3c00 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3c18 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff3c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff3c70 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3ce0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3c70) get_ref 0x0~2000 Blob(0x558e83ff3c70 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3ce0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3c70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e83ff3c70 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3ce0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3cf8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff3d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff3d50 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3dc0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3d50) get_ref 0x0~2000 Blob(0x558e83ff3d50 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff3dc0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff3d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558e83ff3d50 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3dc0 sbid 0x0)) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3dd8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x2c000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e8402e108) dirty_range mark inline shard dirty 2019-12-07T19:33:58.756+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 0x0~2c000 = 0 2019-12-07T19:33:58.756+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.756+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.756+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:58.756+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c57800 cost 361846 (45 ios * 4000 + 181846 bytes) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c57800 onodes 0x558e8402e000 shared_blobs 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e8402e108) update 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e8402e108) update inline shard 421 bytes from 22 extents 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:4986b721:::smithi13140796-43:head# is 840 (413 bytes onode + 2 bytes spanning blobs + 425 bytes inline extents) 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c57800 allocated 0x[1ea000~2c000] released 0x[] 2019-12-07T19:33:58.756+0000 7fe2caa6c700 10 freelist allocate 0x1ea000~2c000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 20 freelist _xor first_key 0x180000 last_key 0x200000 2019-12-07T19:33:58.756+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 prepare 2019-12-07T19:33:58.756+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83c57800 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 lua=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_applied version 21'2 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] eval_repop repgather(0x558e81123b00 21'2 rep_tid=1 committed?=0 r=0) 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81943ce0 finish 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a580 prio 196 cost 0 latency 0.002927 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a580 finish 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e818058c0 prio 196 cost 0 latency 0.002430 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e818058c0 finish 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:128 1.2s0 1.84ed6192 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) queued 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:128 1.2s0 1.84ed6192 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) pg 0x558e81075000 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a6e0 prio 63 cost 63 latency 0.006865 osd_op(client.4540.0:128 1.2s0 1.84ed6192 (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:128 1.2s0 1.84ed6192 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x558e83dd0900 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x558e83dd0900 1:4986b721:::smithi13140796-43:head rwstate(excl n=1 w=0) oi: 1:4986b721:::smithi13140796-43:head(21'2 client.4540.0:127 dirty|data_digest s 720896 uv 2 dd 2bc8c113 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e811239e0 snapset: 0=[]:{} 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:4986b721:::smithi13140796-43:head @head oi=1:4986b721:::smithi13140796-43:head(21'2 client.4540.0:127 dirty|data_digest s 720896 uv 2 dd 2bc8c113 alloc_hint [0 0 0]) 2019-12-07T19:33:58.757+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:4986b721:::smithi13140796-43:head(21'2 client.4540.0:127 dirty|data_digest s 720896 uv 2 dd 2bc8c113 alloc_hint [0 0 0]) 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a6e0 finish 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:129 1.2s0 1.84ed6192 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:129 1.2s0 1.84ed6192 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a2c0 prio 63 cost 0 latency 0.006969 osd_op(client.4540.0:129 1.2s0 1.84ed6192 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:129 1.2s0 1.84ed6192 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x558e83dd0900 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x558e83dd0900 1:4986b721:::smithi13140796-43:head rwstate(excl n=1 w=1) oi: 1:4986b721:::smithi13140796-43:head(21'2 client.4540.0:127 dirty|data_digest s 720896 uv 2 dd 2bc8c113 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e811239e0 snapset: 0=[]:{} 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:4986b721:::smithi13140796-43:head @head oi=1:4986b721:::smithi13140796-43:head(21'2 client.4540.0:127 dirty|data_digest s 720896 uv 2 dd 2bc8c113 alloc_hint [0 0 0]) 2019-12-07T19:33:58.757+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:4986b721:::smithi13140796-43:head(21'2 client.4540.0:127 dirty|data_digest s 720896 uv 2 dd 2bc8c113 alloc_hint [0 0 0]) 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.757+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a2c0 finish 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.757+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 aio_wait 2019-12-07T19:33:58.757+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c57500 2019-12-07T19:33:58.757+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 io_done 2019-12-07T19:33:58.757+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 aio_wait 2019-12-07T19:33:58.757+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c57800 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.757+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 io_done 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2325 < min_flush_size 524288 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3900 had 1 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 2 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2348 < min_flush_size 524288 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2348 < min_flush_size 524288 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3994e~92c to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3994e 0x3994e~92c to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0x994e 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x94e 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x27a and padding block with 0xd86 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3a27a 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000153132s (0.000047337s flush + 0.000105795s kv commit) 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1581 < min_flush_size 524288 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c57500> 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402e000 had 1 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:33:58.757+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 kv_submitted 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1604 < min_flush_size 524288 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c57500 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1604 < min_flush_size 524288 2019-12-07T19:33:58.757+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 finishing 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c57500 onodes 0x558e83cb3900 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd22b8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10cf0 space 0x558e83dd22b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3a27a~644 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2398 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10c60 space 0x558e83dd2398 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3a27a 0x3a27a~644 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd29b8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10d80 space 0x558e83dd29b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2b08 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10bd0 space 0x558e83dd2b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xa27a 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x27a 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2b78 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10b40 space 0x558e83dd2b78 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x8be and padding block with 0x742 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2c58 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10ab0 space 0x558e83dd2c58 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2e18 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10990 space 0x558e83dd2e18 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2e88 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10900 space 0x558e83dd2e88 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3198 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10a20 space 0x558e83dd3198 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd32e8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10e10 space 0x558e83dd32e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd33c8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10630 space 0x558e83dd33c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd34a8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10f30 space 0x558e83dd34a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3588 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10090 space 0x558e83dd3588 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3668 in 0x558e802195e0) _finish_write discard buffer(0x558e83d103f0 space 0x558e83dd3668 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3748 in 0x558e802195e0) _finish_write discard buffer(0x558e83d105a0 space 0x558e83dd3748 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3828 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10510 space 0x558e83dd3828 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3908 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10360 space 0x558e83dd3908 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd39e8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11320 space 0x558e83dd39e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3ac8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d113b0 space 0x558e83dd3ac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3ba8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11440 space 0x558e83dd3ba8 0x0~1000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57500 done 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57800 kv_submitted 2019-12-07T19:33:58.757+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c57500 [] 2019-12-07T19:33:58.757+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.757+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3a8be 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.757+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000163530s (0.000001363s flush + 0.000162167s kv commit) 2019-12-07T19:33:58.757+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c57800> 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.757+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 kv_submitted 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c57800 2019-12-07T19:33:58.757+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 finishing 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c57800 onodes 0x558e8402e000 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff27f8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d115f0 space 0x558e83ff27f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff2948 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11560 space 0x558e83ff2948 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff2cc8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10870 space 0x558e83ff2cc8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff2d38 in 0x558e802195e0) _finish_write discard buffer(0x558e83d107e0 space 0x558e83ff2d38 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff2da8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d119e0 space 0x558e83ff2da8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff2f68 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11a70 space 0x558e83ff2f68 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff2fd8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11b00 space 0x558e83ff2fd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff3048 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11b90 space 0x558e83ff3048 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff3278 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11c20 space 0x558e83ff3278 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff3358 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11cb0 space 0x558e83ff3358 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff3438 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11d40 space 0x558e83ff3438 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff3518 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11dd0 space 0x558e83ff3518 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff35f8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11e60 space 0x558e83ff35f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff36d8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11ef0 space 0x558e83ff36d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff37b8 in 0x558e802195e0) _finish_write discard buffer(0x558e84032000 space 0x558e83ff37b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff3898 in 0x558e802195e0) _finish_write discard buffer(0x558e84032090 space 0x558e83ff3898 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff3978 in 0x558e802195e0) _finish_write discard buffer(0x558e84032120 space 0x558e83ff3978 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff3a58 in 0x558e802195e0) _finish_write discard buffer(0x558e840321b0 space 0x558e83ff3a58 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff3b38 in 0x558e802195e0) _finish_write discard buffer(0x558e84032240 space 0x558e83ff3b38 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff3c18 in 0x558e802195e0) _finish_write discard buffer(0x558e840322d0 space 0x558e83ff3c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff3cf8 in 0x558e802195e0) _finish_write discard buffer(0x558e84032360 space 0x558e83ff3cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff3dd8 in 0x558e802195e0) _finish_write discard buffer(0x558e840323f0 space 0x558e83ff3dd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57800 done 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:33:58.757+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c57800 [] 2019-12-07T19:33:58.757+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.757+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.759+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 67 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e811f7200 con 0x558e818d7c00 2019-12-07T19:33:58.759+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0b1e0 prio 196 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.759+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.759+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.759+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.759+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.759+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 68 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028400 con 0x558e818d7c00 2019-12-07T19:33:58.759+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.759+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0b340 prio 196 cost 0 latency 0.000006 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.759+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.759+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b1e0 prio 196 cost 0 latency 0.000037 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.759+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.759+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.759+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.759+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b1e0 finish 2019-12-07T19:33:58.759+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.759+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.759+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.759+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b340 prio 196 cost 0 latency 0.000035 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.759+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.759+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b340 finish 2019-12-07T19:33:58.759+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.759+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 22 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028600 con 0x558e818fe400 2019-12-07T19:33:58.759+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0adc0 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.759+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.759+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.759+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.759+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.759+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.759+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0adc0 prio 196 cost 0 latency 0.000032 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.759+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.759+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0adc0 finish 2019-12-07T19:33:58.759+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.759+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 6 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028a00 con 0x558e818d6800 2019-12-07T19:33:58.759+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0ab00 prio 196 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.759+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.759+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.759+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.759+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.759+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.759+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0ab00 prio 196 cost 0 latency 0.000054 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.759+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.759+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0ab00 finish 2019-12-07T19:33:58.759+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.761+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 7 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818a9000 con 0x558e818d6800 2019-12-07T19:33:58.761+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0ac60 prio 196 cost 0 latency 0.000018 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.761+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.761+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.761+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.761+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.761+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.761+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0ac60 prio 196 cost 0 latency 0.000050 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.761+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.761+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0ac60 finish 2019-12-07T19:33:58.761+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.761+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 7 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028c00 con 0x558e818ffc00 2019-12-07T19:33:58.761+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0b4a0 prio 196 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.761+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 7 ==== osd_op(client.4540.0:133 1.2s0 1.88517f82 (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 264+0+753664 (crc 0 0 0) 0x558e83da7080 con 0x558e83d68000 2019-12-07T19:33:58.761+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.761+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0af20 prio 63 cost 753664 latency 0.000015 epoch 21 osd_op(client.4540.0:133 1.2s0 1.88517f82 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.761+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:133 1.2s0 1.88517f82 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 753664 e21) 2019-12-07T19:33:58.761+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.761+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.761+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 8 ==== osd_op(client.4540.0:134 1.2s0 1.88517f82 (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 226+0+63 (crc 0 0 0) 0x558e83da7340 con 0x558e83d68000 2019-12-07T19:33:58.761+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e819434a0 prio 63 cost 63 latency 0.000005 epoch 21 osd_op(client.4540.0:134 1.2s0 1.88517f82 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.761+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.761+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:134 1.2s0 1.88517f82 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) 2019-12-07T19:33:58.761+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.761+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b4a0 prio 196 cost 0 latency 0.000047 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.761+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 9 ==== osd_op(client.4540.0:135 1.2s0 1.88517f82 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 ==== 226+0+0 (crc 0 0 0) 0x558e83da7600 con 0x558e83d68000 2019-12-07T19:33:58.761+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81942dc0 prio 63 cost 0 latency 0.000008 epoch 21 osd_op(client.4540.0:135 1.2s0 1.88517f82 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.761+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:135 1.2s0 1.88517f82 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.761+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.761+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b4a0 finish 2019-12-07T19:33:58.761+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.761+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:133 1.2s0 1.88517f82 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 753664 e21) queued 2019-12-07T19:33:58.761+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.761+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:133 1.2s0 1.88517f82 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 753664 e21) pg 0x558e81075000 2019-12-07T19:33:58.761+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0af20 prio 63 cost 753664 latency 0.000082 osd_op(client.4540.0:133 1.2s0 1.88517f82 (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: osd_op(client.4540.0:133 1.2s0 1.88517f82 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.4540.0:133 1.2s0 1:41fe8a11:::smithi13140796-45:head [append 0~753664 in=753664b,stat] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4540.0:133 1.2s0 1:41fe8a11:::smithi13140796-45:head [append 0~753664 in=753664b,stat] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write may_read -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: obc NOT found in cache: 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.762+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# _ 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:41fe8a11:::smithi13140796-45:head# key 0x80800000000000000141fe8a1121736d'ithi13140796-45!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# _ = -2 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: no obc for soid 1:41fe8a11:::smithi13140796-45:head but can_create 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] create_object_context 0x558e83dd1200 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] populate_obc_watchers 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x558e83dd1200 1:41fe8a11:::smithi13140796-45:head rwstate(none n=0 w=0) oi: 1:41fe8a11:::smithi13140796-45:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) ssc: 0x558e81123c20 snapset: 0=[]:{} 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 1:41fe8a11:::smithi13140796-45:head @head oi=1:41fe8a11:::smithi13140796-45:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.762+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op oi 1:41fe8a11:::smithi13140796-45:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op obc obc(1:41fe8a11:::smithi13140796-45:head(dne) rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 0x558e8122b100 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 1:41fe8a11:::smithi13140796-45:head [append 0~753664 in=753664b,stat] ov 0'0 av 21'3 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op 1:41fe8a11:::smithi13140796-45:head [append 0~753664 in=753664b,stat] 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op append 0~753664 in=753664b 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op 1:41fe8a11:::smithi13140796-45:head [write 0~753664 in=753664b] 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op write 0~753664 in=753664b 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] clean_regions modifiedclean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op stat 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] stat oi has 753664 0.000000 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] make_writeable 1:41fe8a11:::smithi13140796-45:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] setting DIRTY flag 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] make_writeable 1:41fe8a11:::smithi13140796-45:head done, snapset=0=[]:{} 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] finish_ctx 1:41fe8a11:::smithi13140796-45:head 0x558e8122b100 op modify 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] set mtime to 2019-12-07T19:33:58.762116+0000 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] final snapset 0=[]:{} in 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] finish_ctx object 1:41fe8a11:::smithi13140796-45:head marks clean_regions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx alloc reply 0x558e8402e280 result 0 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] calc_trim_to_aggressive limit = 0'0 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op order client.4540 tid 133 (first) 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] new_repop rep_tid 2 on osd_op(client.4540.0:133 1.2s0 1:41fe8a11:::smithi13140796-45:head [append 0~753664 in=753664b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] new_repop: repgather(0x558e81123d40 0'0 rep_tid=2 committed?=0 r=0) 2019-12-07T19:33:58.762+0000 7fe2caa6c700 7 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] issue_repop rep_tid 2 o 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] submit_transaction: op Op(1:41fe8a11:::smithi13140796-45:head v=21'3 tt=0'0 tid=2 reqid=client.4540.0:133 client_op=osd_op(client.4540.0:133 1.2s0 1:41fe8a11:::smithi13140796-45:head [append 0~753664 in=753664b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: Getting attr on 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: not in cache 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:41fe8a11:::smithi13140796-45:head# key 0x80800000000000000141fe8a1121736d'ithi13140796-45!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): 1:41fe8a11:::smithi13140796-45:head projected size 753664 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] start_rmw: Op(1:41fe8a11:::smithi13140796-45:head v=21'3 tt=0'0 tid=2 reqid=client.4540.0:133 client_op=osd_op(client.4540.0:133 1.2s0 1:41fe8a11:::smithi13140796-45:head [append 0~753664 in=753664b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41fe8a11:::smithi13140796-45:head=[0~753664]}) 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_state_to_reads: Op(1:41fe8a11:::smithi13140796-45:head v=21'3 tt=0'0 tid=2 reqid=client.4540.0:133 client_op=osd_op(client.4540.0:133 1.2s0 1:41fe8a11:::smithi13140796-45:head [append 0~753664 in=753664b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41fe8a11:::smithi13140796-45:head=[0~753664]}) 2019-12-07T19:33:58.762+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: starting commit on Op(1:41fe8a11:::smithi13140796-45:head v=21'3 tt=0'0 tid=2 reqid=client.4540.0:133 client_op=osd_op(client.4540.0:133 1.2s0 1:41fe8a11:::smithi13140796-45:head [append 0~753664 in=753664b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41fe8a11:::smithi13140796-45:head=[0~753664]}) 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:41df217a:::smithi13140796-39:head)[ Extent(0~638976:1) Extents(1:41fe8a11:::smithi13140796-45:head)[ Extent(0~753664:3) Extents(1:4986b721:::smithi13140796-43:head)[ Extent(0~720896:2) ) 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] generate_transactions: 1:41fe8a11:::smithi13140796-45:head, current size is 0 buffers are {0~753664(753664)} 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): new_size start 0 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): adding buffer_update 0,753664 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): to_append: {0~753664(753664)} 2019-12-07T19:33:58.762+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): appending 0~753664 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] encode_and_write: 1:41fe8a11:::smithi13140796-45:head new_size 753664 2019-12-07T19:33:58.763+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 34 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e81239c00 con 0x558e818d7000 2019-12-07T19:33:58.763+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0b1e0 prio 196 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.763+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): 1:41fe8a11:::smithi13140796-45:head resetting hinfo to logical size 753664 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] operator(): marking append 0 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: ExtentCache( Extents(1:41df217a:::smithi13140796-39:head)[ Extent(0~638976:1) Extents(1:41fe8a11:::smithi13140796-45:head)[ Extent(0~753664:3) Extents(1:4986b721:::smithi13140796-43:head)[ Extent(0~720896:2) ) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: written: {1:41fe8a11:::smithi13140796-45:head={0~753664(753664)}} 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: op: Op(1:41fe8a11:::smithi13140796-45:head v=21'3 tt=0'0 tid=2 reqid=client.4540.0:133 client_op=osd_op(client.4540.0:133 1.2s0 1:41fe8a11:::smithi13140796-45:head [append 0~753664 in=753664b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41fe8a11:::smithi13140796-45:head=[0~753664]}) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: written_set: {1:41fe8a11:::smithi13140796-45:head=[0~753664]} 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_reads_to_commit: 1:41fe8a11:::smithi13140796-45:head,{0~753664(753664)} 2019-12-07T19:33:58.763+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 21/18 ECSubWrite(tid=2, reqid=client.4540.0:133, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e83eda000 con 0x558e818d7c00 2019-12-07T19:33:58.763+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 21/18 ECSubWrite(tid=2, reqid=client.4540.0:133, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e83eda800 con 0x558e818d7000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 21/18 ECSubWrite(tid=2, reqid=client.4540.0:133, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e83edb000 con 0x558e818fe400 2019-12-07T19:33:58.763+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s4 21/18 ECSubWrite(tid=2, reqid=client.4540.0:133, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e83edb800 con 0x558e818d6800 2019-12-07T19:33:58.763+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=2, reqid=client.4540.0:133, at_version=21'3, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e83edc000 con 0x558e818ffc00 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 earliest_dup_version = 0 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 trim 21'3 (0'0) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:133 2019-12-07T19:33:58.762116+0000 0 ObjectCleanRegions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.763+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log log((0'0,21'2], crt=0'0) [21'3 (0'0) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:133 2019-12-07T19:33:58.762116+0000 0 ObjectCleanRegions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.763+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] add_log_entry 21'3 (0'0) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:133 2019-12-07T19:33:58.762116+0000 0 ObjectCleanRegions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.763+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log approx pg log length = 3 2019-12-07T19:33:58.763+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.763+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.763+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.763+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e8195f800 seq 8 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:41fe8a11:::smithi13140796-45:head# key 0x80800000000000000141fe8a1121736d'ithi13140796-45!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.763+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1102 2019-12-07T19:33:58.763+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# = 0 2019-12-07T19:33:58.763+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 2 keys 2019-12-07T19:33:58.763+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 2 keys = 0 2019-12-07T19:33:58.763+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.763+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.763+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 0#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.763+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~2e000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8195f800 23 blobs 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.763+0000 7fe2caa6c700 10 fbmap_alloc 0x558e80233300 allocate 0x2e000/1000,2e000,0 2019-12-07T19:33:58.763+0000 7fe2caa6c700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x216000~2e000/1000,2e000,0 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x216000~2e000] 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e840308c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84030930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e840308c0 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84030930 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e840308c0) get_ref 0x0~2000 Blob(0x558e840308c0 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84030930 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e840308c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e840308c0 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84030930 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84030948 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84030850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e840307e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84030850 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840307e0 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e84030850) get_ref 0x0~2000 Blob(0x558e84030850 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840307e0 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e84030850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e84030850 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840307e0 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e840307f8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84031110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031260 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031110 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031260) get_ref 0x0~2000 Blob(0x558e84031260 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031110 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031260) get_ref init 0x2000, 1000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e84031260 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031110 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031128 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e840312d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84031180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e840312d0 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031180 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e840312d0) get_ref 0x0~2000 Blob(0x558e840312d0 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031180 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e840312d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e840312d0 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031180 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031198 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e840311f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031340 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840311f0 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031340) get_ref 0x0~2000 Blob(0x558e84031340 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840311f0 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e84031340 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840311f0 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031208 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e840313b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84031420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e840313b0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031420 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e840313b0) get_ref 0x0~2000 Blob(0x558e840313b0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031420 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e840313b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e840313b0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031420 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031438 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84031500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031490 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031500 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031490) get_ref 0x0~2000 Blob(0x558e84031490 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031500 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e84031490 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031500 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031518 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e840315e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031570 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840315e0 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031570) get_ref 0x0~2000 Blob(0x558e84031570 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840315e0 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e84031570 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840315e0 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e840315f8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e840316c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031650 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840316c0 sbid 0x0)) 2019-12-07T19:33:58.763+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031650) get_ref 0x0~2000 Blob(0x558e84031650 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840316c0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031650) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e84031650 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840316c0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e840316d8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e840317a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031730 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840317a0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031730) get_ref 0x0~2000 Blob(0x558e84031730 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840317a0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031730) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e84031730 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840317a0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e840317b8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff29a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031810 blob([0x22a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff29a0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031810) get_ref 0x0~2000 Blob(0x558e84031810 blob([0x22a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff29a0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031810) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e84031810 blob([0x22a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff29a0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff29b8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ff2c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ff25b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ff2c40 blob([0x22c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff25b0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff2c40) get_ref 0x0~2000 Blob(0x558e83ff2c40 blob([0x22c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ff25b0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ff2c40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83ff2c40 blob([0x22c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff25b0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff25c8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e840318f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031880 blob([0x22e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840318f0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031880) get_ref 0x0~2000 Blob(0x558e84031880 blob([0x22e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840318f0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e84031880 blob([0x22e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840318f0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031908 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e840319d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031960 blob([0x230000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840319d0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031960) get_ref 0x0~2000 Blob(0x558e84031960 blob([0x230000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840319d0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031960) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 35 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818a8600 con 0x558e818d7000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e84031960 blob([0x230000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840319d0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e840319e8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0adc0 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84031ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031a40 blob([0x232000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031ab0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031a40) get_ref 0x0~2000 Blob(0x558e84031a40 blob([0x232000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031ab0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031a40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e84031a40 blob([0x232000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031ab0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031ac8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84031b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031b20 blob([0x234000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031b90 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031b20) get_ref 0x0~2000 Blob(0x558e84031b20 blob([0x234000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031b90 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031b20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e84031b20 blob([0x234000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031b90 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031ba8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84031c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031c00 blob([0x236000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031c70 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031c00) get_ref 0x0~2000 Blob(0x558e84031c00 blob([0x236000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031c70 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031c00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e84031c00 blob([0x236000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031c70 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031c88 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84031d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031ce0 blob([0x238000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031d50 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031ce0) get_ref 0x0~2000 Blob(0x558e84031ce0 blob([0x238000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031d50 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031ce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e84031ce0 blob([0x238000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031d50 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031d68 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84031e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031dc0 blob([0x23a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031e30 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031dc0) get_ref 0x0~2000 Blob(0x558e84031dc0 blob([0x23a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031e30 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031dc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e84031dc0 blob([0x23a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031e30 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031e48 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84031f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031ea0 blob([0x23c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031f10 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031ea0) get_ref 0x0~2000 Blob(0x558e84031ea0 blob([0x23c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84031f10 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031ea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e84031ea0 blob([0x23c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031f10 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031f28 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84031f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ede000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84031f80 blob([0x23e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ede000 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031f80) get_ref 0x0~2000 Blob(0x558e84031f80 blob([0x23e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ede000 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e84031f80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e84031f80 blob([0x23e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede000 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ede018 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ede070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ede0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ede070 blob([0x240000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ede0e0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ede070) get_ref 0x0~2000 Blob(0x558e83ede070 blob([0x240000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ede0e0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ede070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558e83ede070 blob([0x240000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede0e0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ede0f8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ede150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ede1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ede150 blob([0x242000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ede1c0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ede150) get_ref 0x0~2000 Blob(0x558e83ede150 blob([0x242000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ede1c0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.blob(0x558e83ede150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558e83ede150 blob([0x242000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede1c0 sbid 0x0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ede1d8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x2e000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e8402eb08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 = 0 2019-12-07T19:33:58.764+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.764+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195f800 cost 378038 (47 ios * 4000 + 190038 bytes) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195f800 onodes 0x558e8402ea00 shared_blobs 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e8402eb08) update 0#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e8402eb08) update inline shard 440 bytes from 23 extents 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:41fe8a11:::smithi13140796-45:head# is 859 (413 bytes onode + 2 bytes spanning blobs + 444 bytes inline extents) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195f800 allocated 0x[216000~2e000] released 0x[] 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 freelist allocate 0x216000~2e000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 freelist _xor first_key 0x200000 last_key 0x200000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 prepare 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8195f800 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 lua=21'2 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_applied version 21'3 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] eval_repop repgather(0x558e81123d40 21'3 rep_tid=2 committed?=0 r=0) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0af20 finish 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b1e0 prio 196 cost 0 latency 0.001082 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=0, last_complete=21'1, committed=1, applied=1)) v2 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:41df217a:::smithi13140796-39:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:115 client_op=osd_op(client.4540.0:115 1.2s0 1:41df217a:::smithi13140796-39:head [append 0~638976 in=638976b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:head=[0~638976]}) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=0'0 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] repop_all_committed: repop tid 0 all committed 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] eval_repop repgather(0x558e811238c0 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] commit: repgather(0x558e811238c0 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] log_op_stats osd_op(client.4540.0:115 1.2s0 1:41df217a:::smithi13140796-39:head [append 0~638976 in=638976b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 638976 outb 0 lat 0.015861 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] sending reply on osd_op(client.4540.0:115 1.2s0 1:41df217a:::smithi13140796-39:head [append 0~638976 in=638976b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x558e83c72c80 2019-12-07T19:33:58.764+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(115 smithi13140796-39 [append 0~638976,stat] v21'1 uv1 ondisk = 0) v8 -- 0x558e83c72c80 con 0x558e83d68000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.764+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] publish_stats_to_osd 19:13 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] removing repgather(0x558e811238c0 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] q front is repgather(0x558e811238c0 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] remove_repop repgather(0x558e811238c0 21'1 rep_tid=0 committed?=1 r=0) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] requeue_op 0x558e819438c0 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] requeue_op 0x558e81942b00 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_finish_rmw: Op(1:41df217a:::smithi13140796-39:head v=21'1 tt=0'0 tid=0 reqid=client.4540.0:115 client_op=osd_op(client.4540.0:115 1.2s0 1:41df217a:::smithi13140796-39:head [append 0~638976 in=638976b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:head=[0~638976]}) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] try_finish_rmw: ExtentCache( Extents(1:41df217a:::smithi13140796-39:head)[ Extent(0~638976:1) Extents(1:41fe8a11:::smithi13140796-45:head)[ Extent(0~753664:3) Extents(1:4986b721:::smithi13140796-43:head)[ Extent(0~720896:2) ) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b1e0 finish 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0adc0 prio 196 cost 0 latency 0.000683 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=1, last_complete=21'2, committed=1, applied=1)) v2 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:4986b721:::smithi13140796-43:head v=21'2 tt=0'0 tid=1 reqid=client.4540.0:127 client_op=osd_op(client.4540.0:127 1.2s0 1:4986b721:::smithi13140796-43:head [append 0~720896 in=720896b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4986b721:::smithi13140796-43:head=[0~720896]}) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'1 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] repop_all_committed: repop tid 1 all committed 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] eval_repop repgather(0x558e81123b00 21'2 rep_tid=1 committed?=1 r=0) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] commit: repgather(0x558e81123b00 21'2 rep_tid=1 committed?=1 r=0) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] log_op_stats osd_op(client.4540.0:127 1.2s0 1:4986b721:::smithi13140796-43:head [append 0~720896 in=720896b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 720896 outb 0 lat 0.014443 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] sending reply on osd_op(client.4540.0:127 1.2s0 1:4986b721:::smithi13140796-43:head [append 0~720896 in=720896b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x558e81a03900 2019-12-07T19:33:58.764+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(127 smithi13140796-43 [append 0~720896,stat] v21'2 uv2 ondisk = 0) v8 -- 0x558e81a03900 con 0x558e83d68000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.764+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] publish_stats_to_osd 21:14 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] removing repgather(0x558e81123b00 21'2 rep_tid=1 committed?=1 r=0) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] q front is repgather(0x558e81123b00 21'2 rep_tid=1 committed?=1 r=0) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] remove_repop repgather(0x558e81123b00 21'2 rep_tid=1 committed?=1 r=0) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] requeue_op 0x558e83b0a2c0 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] requeue_op 0x558e83b0a6e0 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] try_finish_rmw: Op(1:4986b721:::smithi13140796-43:head v=21'2 tt=0'0 tid=1 reqid=client.4540.0:127 client_op=osd_op(client.4540.0:127 1.2s0 1:4986b721:::smithi13140796-43:head [append 0~720896 in=720896b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4986b721:::smithi13140796-43:head=[0~720896]}) 2019-12-07T19:33:58.764+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] try_finish_rmw: ExtentCache( Extents(1:41fe8a11:::smithi13140796-45:head)[ Extent(0~753664:3) Extents(1:4986b721:::smithi13140796-43:head)[ Extent(0~720896:2) ) 2019-12-07T19:33:58.764+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) queued 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0adc0 finish 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) pg 0x558e81075000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a6e0 prio 63 cost 63 latency 0.014487 osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 pg pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] _handle_message: osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] do_op: op osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] do_op osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] get_object_context: found obc in cache: 0x558e83dd0900 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] get_object_context: 0x558e83dd0900 1:4986b721:::smithi13140796-43:head rwstate(none n=0 w=0) oi: 1:4986b721:::smithi13140796-43:head(21'2 client.4540.0:127 dirty|data_digest s 720896 uv 2 dd 2bc8c113 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e811239e0 snapset: 0=[]:{} 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] find_object_context 1:4986b721:::smithi13140796-43:head @head oi=1:4986b721:::smithi13140796-43:head(21'2 client.4540.0:127 dirty|data_digest s 720896 uv 2 dd 2bc8c113 alloc_hint [0 0 0]) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] do_op oi 1:4986b721:::smithi13140796-43:head(21'2 client.4540.0:127 dirty|data_digest s 720896 uv 2 dd 2bc8c113 alloc_hint [0 0 0]) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] do_op obc obc(1:4986b721:::smithi13140796-43:head rwstate(write n=1 w=0)) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] execute_ctx 0x558e80ff3800 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] execute_ctx 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] ov 21'2 av 21'4 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] do_osd_op 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] do_osd_op setxattr _header (56) in=63b 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] make_writeable 1:4986b721:::smithi13140796-43:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] make_writeable 1:4986b721:::smithi13140796-43:head done, snapset=0=[]:{} 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] finish_ctx 1:4986b721:::smithi13140796-43:head 0x558e80ff3800 op modify 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] set mtime to 2019-12-07T19:33:58.750475+0000 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] final snapset 0=[]:{} in 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] finish_ctx object 1:4986b721:::smithi13140796-43:head marks clean_regions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] execute_ctx alloc reply 0x558e811dcc80 result 0 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] calc_trim_to_aggressive limit = 0'0 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] op order client.4540 tid 128 last was 127 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] new_repop rep_tid 3 on osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] new_repop: repgather(0x558e81123b00 0'0 rep_tid=3 committed?=0 r=0) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 7 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] issue_repop rep_tid 3 o 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] submit_transaction: op Op(1:4986b721:::smithi13140796-43:head v=21'4 tt=0'0 tid=3 reqid=client.4540.0:128 client_op=osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] get_hash_info: Getting attr on 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] get_hash_info: not in cache 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] get_hash_info: found on disk, size 180224 2019-12-07T19:33:58.764+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key = 0 2019-12-07T19:33:58.764+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] operator(): 1:4986b721:::smithi13140796-43:head projected size 720896 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] start_rmw: Op(1:4986b721:::smithi13140796-43:head v=21'4 tt=0'0 tid=3 reqid=client.4540.0:128 client_op=osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4986b721:::smithi13140796-43:head=[]}) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] try_state_to_reads: Op(1:4986b721:::smithi13140796-43:head v=21'4 tt=0'0 tid=3 reqid=client.4540.0:128 client_op=osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4986b721:::smithi13140796-43:head=[]}) 2019-12-07T19:33:58.764+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: starting commit on Op(1:4986b721:::smithi13140796-43:head v=21'4 tt=0'0 tid=3 reqid=client.4540.0:128 client_op=osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4986b721:::smithi13140796-43:head=[]}) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: ExtentCache( Extents(1:41fe8a11:::smithi13140796-45:head)[ Extent(0~753664:3) ) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] generate_transactions: 1:4986b721:::smithi13140796-43:head, current size is 720896 buffers are {} 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] operator(): new_size start 720896 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] operator(): to_append: {} 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] operator(): 1:4986b721:::smithi13140796-43:head resetting hinfo to logical size 720896 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: ExtentCache( Extents(1:41fe8a11:::smithi13140796-45:head)[ Extent(0~753664:3) ) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: written: {1:4986b721:::smithi13140796-43:head={}} 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: op: Op(1:4986b721:::smithi13140796-43:head v=21'4 tt=0'0 tid=3 reqid=client.4540.0:128 client_op=osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4986b721:::smithi13140796-43:head=[]}) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: written_set: {1:4986b721:::smithi13140796-43:head=[]} 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: 1:4986b721:::smithi13140796-43:head,{} 2019-12-07T19:33:58.765+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 21/18 ECSubWrite(tid=3, reqid=client.4540.0:128, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558e83edd000 con 0x558e818d7c00 2019-12-07T19:33:58.765+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 21/18 ECSubWrite(tid=3, reqid=client.4540.0:128, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558e83edd800 con 0x558e818d7000 2019-12-07T19:33:58.765+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 21/18 ECSubWrite(tid=3, reqid=client.4540.0:128, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558e83fea000 con 0x558e818fe400 2019-12-07T19:33:58.765+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s4 21/18 ECSubWrite(tid=3, reqid=client.4540.0:128, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558e83fea800 con 0x558e818d6800 2019-12-07T19:33:58.765+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=3, reqid=client.4540.0:128, at_version=21'4, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558e83feb000 con 0x558e818ffc00 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 earliest_dup_version = 0 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 trim 21'4 (21'2) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:128 2019-12-07T19:33:58.750475+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] append_log log((0'0,21'3], crt=0'0) [21'4 (21'2) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:128 2019-12-07T19:33:58.750475+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 lua=21'3 crt=0'0 lcod 21'1 mlcod 21'1 active+clean] add_log_entry 21'4 (21'2) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:128 2019-12-07T19:33:58.750475+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 lua=21'3 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] rollforward: entry=21'1 (0'0) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:115 2019-12-07T19:33:58.731888+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 lua=21'3 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] rollforward: entry=21'2 (0'0) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:127 2019-12-07T19:33:58.750442+0000 0 ObjectCleanRegions clean_offsets: [720896~18446744073708830719], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 lua=21'3 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] append_log approx pg log length = 4 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 lua=21'3 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.765+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e819a1500 seq 9 2019-12-07T19:33:58.765+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 3 keys 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 3 keys = 0 2019-12-07T19:33:58.765+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.765+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1500 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1500 onodes 0x558e8402e000 shared_blobs 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e8402e108) update 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:4986b721:::smithi13140796-43:head# is 912 (485 bytes onode + 2 bytes spanning blobs + 425 bytes inline extents) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 prepare 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1500 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1500 blocked by 0x558e8195f800 aio_wait 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 lua=21'3 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] op_applied version 21'4 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] eval_repop repgather(0x558e81123b00 21'4 rep_tid=3 committed?=0 r=0) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a6e0 finish 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) queued 2019-12-07T19:33:58.765+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.765+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.765+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0a2c0 prio 63 cost 0 latency 0.015013 osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.765+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.765+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op: op osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.765+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 aio_wait 2019-12-07T19:33:58.765+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195f800 2019-12-07T19:33:58.765+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.765+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 io_done 2019-12-07T19:33:58.765+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 io_done 2019-12-07T19:33:58.765+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.765+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: found obc in cache: 0x558e83dd0900 2019-12-07T19:33:58.765+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: 0x558e83dd0900 1:4986b721:::smithi13140796-43:head rwstate(write n=1 w=0) oi: 1:4986b721:::smithi13140796-43:head(21'4 client.4540.0:128 dirty|data_digest s 720896 uv 4 dd 2bc8c113 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e811239e0 snapset: 0=[]:{} 2019-12-07T19:33:58.765+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] find_object_context 1:4986b721:::smithi13140796-43:head @head oi=1:4986b721:::smithi13140796-43:head(21'4 client.4540.0:128 dirty|data_digest s 720896 uv 4 dd 2bc8c113 alloc_hint [0 0 0]) 2019-12-07T19:33:58.765+0000 7fe2c6a64700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op oi 1:4986b721:::smithi13140796-43:head(21'4 client.4540.0:128 dirty|data_digest s 720896 uv 4 dd 2bc8c113 alloc_hint [0 0 0]) 2019-12-07T19:33:58.765+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.765+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0a2c0 finish 2019-12-07T19:33:58.765+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1571 < min_flush_size 524288 2019-12-07T19:33:58.765+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.765+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ea00 had 1 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 2 2019-12-07T19:33:58.765+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3523 < min_flush_size 524288 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) pg 0x558e81075000 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402e000 had 1 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:33:58.765+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3546 < min_flush_size 524288 2019-12-07T19:33:58.765+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3546 < min_flush_size 524288 2019-12-07T19:33:58.765+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.765+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3a8be~dda to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81942b00 prio 63 cost 63 latency 0.016702 osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 pg pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.765+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3a8be 0x3a8be~dda to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xa8be 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x8be 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x698 and padding block with 0x968 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op: op osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: found obc in cache: 0x558e81222900 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: 0x558e81222900 1:41df217a:::smithi13140796-39:head rwstate(none n=0 w=0) oi: 1:41df217a:::smithi13140796-39:head(21'1 client.4540.0:115 dirty|data_digest s 638976 uv 1 dd b038798b alloc_hint [0 0 0]) exists: 1 ssc: 0x558e811237a0 snapset: 0=[]:{} 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3b698 2019-12-07T19:33:58.765+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.765+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.765+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] find_object_context 1:41df217a:::smithi13140796-39:head @head oi=1:41df217a:::smithi13140796-39:head(21'1 client.4540.0:115 dirty|data_digest s 638976 uv 1 dd b038798b alloc_hint [0 0 0]) 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000169409s (0.000042653s flush + 0.000126756s kv commit) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op oi 1:41df217a:::smithi13140796-39:head(21'1 client.4540.0:115 dirty|data_digest s 638976 uv 1 dd b038798b alloc_hint [0 0 0]) 2019-12-07T19:33:58.765+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op obc obc(1:41df217a:::smithi13140796-39:head rwstate(write n=1 w=0)) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] execute_ctx 0x558e81228e00 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] execute_ctx 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] ov 21'1 av 21'5 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_osd_op 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_osd_op setxattr _header (56) in=63b 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] make_writeable 1:41df217a:::smithi13140796-39:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] make_writeable 1:41df217a:::smithi13140796-39:head done, snapset=0=[]:{} 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] finish_ctx 1:41df217a:::smithi13140796-39:head 0x558e81228e00 op modify 2019-12-07T19:33:58.765+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 23 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e81239c00 con 0x558e818fe400 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] set mtime to 2019-12-07T19:33:58.731948+0000 2019-12-07T19:33:58.765+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a580 prio 196 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.765+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] final snapset 0=[]:{} in 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] finish_ctx object 1:41df217a:::smithi13140796-39:head marks clean_regions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] execute_ctx alloc reply 0x558e83b09680 result 0 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] calc_trim_to_aggressive limit = 21'2 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] op order client.4540 tid 116 last was 115 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] new_repop rep_tid 4 on osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] new_repop: repgather(0x558e811238c0 0'0 rep_tid=4 committed?=0 r=0) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 7 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] issue_repop rep_tid 4 o 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.765+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 8 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84029c00 con 0x558e818d6800 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] submit_transaction: op Op(1:41df217a:::smithi13140796-39:head v=21'5 tt=0'0 tid=4 reqid=client.4540.0:116 client_op=osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.765+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0adc0 prio 196 cost 0 latency 0.000013 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.765+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_hash_info: Getting attr on 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_hash_info: not in cache 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.765+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 9 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e810fce00 con 0x558e818d6800 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:33:58.765+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0b4a0 prio 196 cost 0 latency 0.000007 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.765+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_hash_info: found on disk, size 159744 2019-12-07T19:33:58.765+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key = 0 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] operator(): 1:41df217a:::smithi13140796-39:head projected size 638976 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] start_rmw: Op(1:41df217a:::smithi13140796-39:head v=21'5 tt=0'0 tid=4 reqid=client.4540.0:116 client_op=osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] try_state_to_reads: Op(1:41df217a:::smithi13140796-39:head v=21'5 tt=0'0 tid=4 reqid=client.4540.0:116 client_op=osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: starting commit on Op(1:41df217a:::smithi13140796-39:head v=21'5 tt=0'0 tid=4 reqid=client.4540.0:116 client_op=osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: ExtentCache( Extents(1:41fe8a11:::smithi13140796-45:head)[ Extent(0~753664:3) ) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] generate_transactions: 1:41df217a:::smithi13140796-39:head, current size is 638976 buffers are {} 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] operator(): new_size start 638976 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] operator(): to_append: {} 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] operator(): 1:41df217a:::smithi13140796-39:head resetting hinfo to logical size 638976 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: ExtentCache( Extents(1:41fe8a11:::smithi13140796-45:head)[ Extent(0~753664:3) ) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: written: {1:41df217a:::smithi13140796-39:head={}} 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: op: Op(1:41df217a:::smithi13140796-39:head v=21'5 tt=0'0 tid=4 reqid=client.4540.0:116 client_op=osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:33:58.765+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 24 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028a00 con 0x558e818fe400 2019-12-07T19:33:58.765+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a000 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: written_set: {1:41df217a:::smithi13140796-39:head=[]} 2019-12-07T19:33:58.765+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] try_reads_to_commit: 1:41df217a:::smithi13140796-39:head,{} 2019-12-07T19:33:58.765+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 69 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818efe00 con 0x558e818d7c00 2019-12-07T19:33:58.765+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0b600 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.765+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 21/18 ECSubWrite(tid=4, reqid=client.4540.0:116, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558e83feb800 con 0x558e818d7c00 2019-12-07T19:33:58.765+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 21/18 ECSubWrite(tid=4, reqid=client.4540.0:116, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558e83fec000 con 0x558e818d7000 2019-12-07T19:33:58.765+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 8 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028000 con 0x558e818ffc00 2019-12-07T19:33:58.765+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0b760 prio 196 cost 0 latency 0.000008 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.765+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 21/18 ECSubWrite(tid=4, reqid=client.4540.0:116, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558e83fec800 con 0x558e818fe400 2019-12-07T19:33:58.765+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.765+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s4 21/18 ECSubWrite(tid=4, reqid=client.4540.0:116, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558e83fed000 con 0x558e818d6800 2019-12-07T19:33:58.765+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=4, reqid=client.4540.0:116, at_version=21'5, trim_to=0'0, roll_forward_to=21'2)) v2 -- 0x558e83fed800 con 0x558e818ffc00 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 earliest_dup_version = 0 2019-12-07T19:33:58.765+0000 7fe2caa6c700 20 trim 21'5 (21'1) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:116 2019-12-07T19:33:58.731948+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] append_log log((0'0,21'4], crt=21'2) [21'5 (21'1) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:116 2019-12-07T19:33:58.731948+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 lua=21'4 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] add_log_entry 21'5 (21'1) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:116 2019-12-07T19:33:58.731948+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 lua=21'4 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] append_log approx pg log length = 5 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 lua=21'4 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.766+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e81a0b800 seq 10 2019-12-07T19:33:58.766+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 3 keys 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 3 keys = 0 2019-12-07T19:33:58.766+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.766+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b800 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b800 onodes 0x558e83cb3900 shared_blobs 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e83cb3a08) update 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:41df217a:::smithi13140796-39:head# is 871 (485 bytes onode + 2 bytes spanning blobs + 384 bytes inline extents) 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 70 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e80264c00 con 0x558e818d7c00 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0b8c0 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 prepare 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b800 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 io_done 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 lua=21'4 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] op_applied version 21'5 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] eval_repop repgather(0x558e811238c0 21'5 rep_tid=4 committed?=0 r=0) 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81942b00 finish 2019-12-07T19:33:58.766+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.766+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.766+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1911 < min_flush_size 524288 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.766+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3900 had 1 2019-12-07T19:33:58.766+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1934 < min_flush_size 524288 2019-12-07T19:33:58.766+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1934 < min_flush_size 524288 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.766+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3b698~78e to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.766+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3b698 0x3b698~78e to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e819438c0 prio 63 cost 0 latency 0.017360 osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.766+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xb698 2019-12-07T19:33:58.766+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x698 2019-12-07T19:33:58.766+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.766+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xe26 and padding block with 0x1da 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op: op osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: found obc in cache: 0x558e81222900 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: 0x558e81222900 1:41df217a:::smithi13140796-39:head rwstate(write n=1 w=0) oi: 1:41df217a:::smithi13140796-39:head(21'5 client.4540.0:116 dirty|data_digest s 638976 uv 5 dd b038798b alloc_hint [0 0 0]) exists: 1 ssc: 0x558e811237a0 snapset: 0=[]:{} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] find_object_context 1:41df217a:::smithi13140796-39:head @head oi=1:41df217a:::smithi13140796-39:head(21'5 client.4540.0:116 dirty|data_digest s 638976 uv 5 dd b038798b alloc_hint [0 0 0]) 2019-12-07T19:33:58.766+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op oi 1:41df217a:::smithi13140796-39:head(21'5 client.4540.0:116 dirty|data_digest s 638976 uv 5 dd b038798b alloc_hint [0 0 0]) 2019-12-07T19:33:58.766+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3be26 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.766+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.766+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e819438c0 finish 2019-12-07T19:33:58.766+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.766+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.766+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000114269s (0.000000143s flush + 0.000114126s kv commit) 2019-12-07T19:33:58.766+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a580 prio 196 cost 0 latency 0.000642 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a580 finish 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0adc0 prio 196 cost 0 latency 0.000596 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0adc0 finish 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b760 prio 196 cost 0 latency 0.000402 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b760 finish 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b600 prio 196 cost 0 latency 0.000481 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b600 finish 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a000 prio 196 cost 0 latency 0.000523 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a000 finish 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 9 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84029e00 con 0x558e818ffc00 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0ba20 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b4a0 prio 196 cost 0 latency 0.000692 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b4a0 finish 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195f800,0x558e819a1500,0x558e81a0b800> 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.766+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 kv_submitted 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195f800 2019-12-07T19:33:58.766+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 finishing 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195f800 onodes 0x558e8402ea00 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ede018 in 0x558e802195e0) _finish_write discard buffer(0x558e83d107e0 space 0x558e83ede018 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ede0f8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d119e0 space 0x558e83ede0f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ede1d8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11a70 space 0x558e83ede1d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff25c8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d105a0 space 0x558e83ff25c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ff29b8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d103f0 space 0x558e83ff29b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e840307f8 in 0x558e802195e0) _finish_write discard buffer(0x558e83c8e3f0 space 0x558e840307f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84030948 in 0x558e802195e0) _finish_write discard buffer(0x558e83c8fb90 space 0x558e84030948 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84031128 in 0x558e802195e0) _finish_write discard buffer(0x558e83c8f290 space 0x558e84031128 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84031198 in 0x558e802195e0) _finish_write discard buffer(0x558e83c8f170 space 0x558e84031198 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84031208 in 0x558e802195e0) _finish_write discard buffer(0x558e83c8e480 space 0x558e84031208 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 25 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84029000 con 0x558e818fe400 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84031438 in 0x558e802195e0) _finish_write discard buffer(0x558e83c8ecf0 space 0x558e84031438 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84031518 in 0x558e802195e0) _finish_write discard buffer(0x558e83c8e990 space 0x558e84031518 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a160 prio 196 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e840315f8 in 0x558e802195e0) _finish_write discard buffer(0x558e83c8fc20 space 0x558e840315f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e840316d8 in 0x558e802195e0) _finish_write discard buffer(0x558e83c8eab0 space 0x558e840316d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e840317b8 in 0x558e802195e0) _finish_write discard buffer(0x558e81171050 space 0x558e840317b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84031908 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10510 space 0x558e84031908 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b8c0 prio 196 cost 0 latency 0.000372 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 71 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83fef000 con 0x558e818d7c00 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e840319e8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10360 space 0x558e840319e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84031ac8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11320 space 0x558e84031ac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0bb80 prio 196 cost 0 latency 0.000010 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84031ba8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d113b0 space 0x558e84031ba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84031c88 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11440 space 0x558e84031c88 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84031d68 in 0x558e802195e0) _finish_write discard buffer(0x558e83d115f0 space 0x558e84031d68 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84031e48 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11560 space 0x558e84031e48 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b8c0 finish 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84031f28 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10870 space 0x558e84031f28 0x0~2000 writing nocache) 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f800 done 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1500 kv_submitted 2019-12-07T19:33:58.766+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195f800 [] 2019-12-07T19:33:58.766+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 0 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818ffc00 msgr2=0x558e818e7180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2019-12-07T19:33:58.766+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 kv_submitted 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1500 2019-12-07T19:33:58.766+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 finishing 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1500 onodes 0x558e8402e000 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1500 done 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 kv_submitted 2019-12-07T19:33:58.766+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1500 [] 2019-12-07T19:33:58.766+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 kv_submitted 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b800 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.766+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 finishing 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b800 onodes 0x558e83cb3900 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 done 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 10 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e80264000 con 0x558e818ffc00 2019-12-07T19:33:58.766+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b800 [] 2019-12-07T19:33:58.766+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81942420 prio 196 cost 0 latency 0.000007 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818ffc00 msgr2=0x558e818e7180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 96 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0ba20 prio 196 cost 0 latency 0.000111 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818ffc00 msgr2=0x558e818e7180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818ffc00 0x558e818e7180 unknown :-1 s=READY pgs=7 cs=0 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0ba20 finish 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818ffc00 0x558e818e7180 unknown :-1 s=READY pgs=7 cs=0 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a160 prio 196 cost 0 latency 0.000101 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 10 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83feee00 con 0x558e818d6800 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a160 finish 2019-12-07T19:33:58.766+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0ab00 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0bb80 prio 196 cost 0 latency 0.000147 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0bb80 finish 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81942420 prio 196 cost 0 latency 0.000153 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81942420 finish 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0ab00 prio 196 cost 0 latency 0.000130 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:134 1.2s0 1.88517f82 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) queued 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e83d40800 0x558e83ae2100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0ab00 finish 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:135 1.2s0 1.88517f82 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:134 1.2s0 1.88517f82 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e83d40800 0x558e83ae2100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e819434a0 prio 63 cost 63 latency 0.004822 osd_op(client.4540.0:134 1.2s0 1.88517f82 (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: osd_op(client.4540.0:134 1.2s0 1.88517f82 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op: op osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: found obc in cache: 0x558e83dd1200 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: 0x558e83dd1200 1:41fe8a11:::smithi13140796-45:head rwstate(excl n=1 w=0) oi: 1:41fe8a11:::smithi13140796-45:head(21'3 client.4540.0:133 dirty|data_digest s 753664 uv 3 dd 1117d0c7 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e81123c20 snapset: 0=[]:{} 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] find_object_context 1:41fe8a11:::smithi13140796-45:head @head oi=1:41fe8a11:::smithi13140796-45:head(21'3 client.4540.0:133 dirty|data_digest s 753664 uv 3 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:33:58.766+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op oi 1:41fe8a11:::smithi13140796-45:head(21'3 client.4540.0:133 dirty|data_digest s 753664 uv 3 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 10 osd.0 21 ms_handle_authentication new session 0x558e818e8ea0 con 0x558e83d40800 entity osd.6 addr 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.766+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e819434a0 finish 2019-12-07T19:33:58.766+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:135 1.2s0 1.88517f82 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 10 osd.0 21 ms_handle_authentication session 0x558e818e8ea0 osd.6 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81942dc0 prio 63 cost 0 latency 0.004929 osd_op(client.4540.0:135 1.2s0 1.88517f82 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: osd_op(client.4540.0:135 1.2s0 1.88517f82 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op: op osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: found obc in cache: 0x558e83dd1200 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] get_object_context: 0x558e83dd1200 1:41fe8a11:::smithi13140796-45:head rwstate(excl n=1 w=1) oi: 1:41fe8a11:::smithi13140796-45:head(21'3 client.4540.0:133 dirty|data_digest s 753664 uv 3 dd 1117d0c7 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e81123c20 snapset: 0=[]:{} 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e83d40800 0x558e83ae2100 crc :-1 s=SESSION_ACCEPTING pgs=16 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x558e818ffc00 2019-12-07T19:33:58.766+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e83d40800 0x558e83ae2100 crc :-1 s=SESSION_ACCEPTING pgs=16 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:58.766+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] find_object_context 1:41fe8a11:::smithi13140796-45:head @head oi=1:41fe8a11:::smithi13140796-45:head(21'3 client.4540.0:133 dirty|data_digest s 753664 uv 3 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:33:58.766+0000 7fe2c6a64700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op oi 1:41fe8a11:::smithi13140796-45:head(21'3 client.4540.0:133 dirty|data_digest s 753664 uv 3 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:33:58.766+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.767+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81942dc0 finish 2019-12-07T19:33:58.767+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.767+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818ffc00 0x558e818e7180 unknown :-1 s=READY pgs=16 cs=1 l=0 rx=0 tx=0).ready entity=osd.6 client_cookie=66df44187c63ea59 server_cookie=2175ee4a8507a6a8 in_seq=10 out_seq=11 2019-12-07T19:33:58.768+0000 7fe2d4a80700 2 osd.0 21 ms_handle_reset con 0x558e83d40800 session 0x558e818e8ea0 2019-12-07T19:33:58.769+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 36 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028c00 con 0x558e818d7000 2019-12-07T19:33:58.769+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0b340 prio 196 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.769+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.769+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.769+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.769+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.769+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.769+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b340 prio 196 cost 0 latency 0.000058 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] 2019-12-07T19:33:58.769+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=2, last_complete=21'3, committed=1, applied=1)) v2 2019-12-07T19:33:58.769+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 37 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028e00 con 0x558e818d7000 2019-12-07T19:33:58.769+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0bce0 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.769+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:41fe8a11:::smithi13140796-45:head v=21'3 tt=0'0 tid=2 reqid=client.4540.0:133 client_op=osd_op(client.4540.0:133 1.2s0 1:41fe8a11:::smithi13140796-45:head [append 0~753664 in=753664b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41fe8a11:::smithi13140796-45:head=[0~753664]}) 2019-12-07T19:33:58.769+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.769+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'2 crt=21'2 lcod 21'1 mlcod 21'1 active+clean] repop_all_committed: repop tid 2 all committed 2019-12-07T19:33:58.769+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.769+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.769+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] eval_repop repgather(0x558e81123d40 21'3 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.769+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] commit: repgather(0x558e81123d40 21'3 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.769+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 38 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84029c00 con 0x558e818d7000 2019-12-07T19:33:58.769+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0be40 prio 196 cost 0 latency 0.000008 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.769+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.769+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] log_op_stats osd_op(client.4540.0:133 1.2s0 1:41fe8a11:::smithi13140796-45:head [append 0~753664 in=753664b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 753664 outb 0 lat 0.007687 2019-12-07T19:33:58.769+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] sending reply on osd_op(client.4540.0:133 1.2s0 1:41fe8a11:::smithi13140796-45:head [append 0~753664 in=753664b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x558e8402e280 2019-12-07T19:33:58.769+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(133 smithi13140796-45 [append 0~753664,stat] v21'3 uv3 ondisk = 0) v8 -- 0x558e8402e280 con 0x558e83d68000 2019-12-07T19:33:58.769+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.769+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] publish_stats_to_osd 21:15 2019-12-07T19:33:58.769+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] removing repgather(0x558e81123d40 21'3 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.769+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] q front is repgather(0x558e81123d40 21'3 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.769+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:33:58.769+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] remove_repop repgather(0x558e81123d40 21'3 rep_tid=2 committed?=1 r=0) 2019-12-07T19:33:58.769+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] requeue_op 0x558e81942dc0 2019-12-07T19:33:58.769+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) shuffled w/ OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.769+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] requeue_op 0x558e819434a0 2019-12-07T19:33:58.769+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) shuffled w/ OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.769+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_finish_rmw: Op(1:41fe8a11:::smithi13140796-45:head v=21'3 tt=0'0 tid=2 reqid=client.4540.0:133 client_op=osd_op(client.4540.0:133 1.2s0 1:41fe8a11:::smithi13140796-45:head [append 0~753664 in=753664b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41fe8a11:::smithi13140796-45:head=[0~753664]}) 2019-12-07T19:33:58.769+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_finish_rmw: ExtentCache( Extents(1:41fe8a11:::smithi13140796-45:head)[ Extent(0~753664:3) ) 2019-12-07T19:33:58.769+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b340 finish 2019-12-07T19:33:58.769+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.769+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.769+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.769+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) pg 0x558e81075000 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e819434a0 prio 63 cost 63 latency 0.007871 osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 pg pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] _handle_message: osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.769+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] do_op: op osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.769+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] do_op osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] get_object_context: found obc in cache: 0x558e83dd1200 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] get_object_context: 0x558e83dd1200 1:41fe8a11:::smithi13140796-45:head rwstate(none n=0 w=0) oi: 1:41fe8a11:::smithi13140796-45:head(21'3 client.4540.0:133 dirty|data_digest s 753664 uv 3 dd 1117d0c7 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e81123c20 snapset: 0=[]:{} 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] find_object_context 1:41fe8a11:::smithi13140796-45:head @head oi=1:41fe8a11:::smithi13140796-45:head(21'3 client.4540.0:133 dirty|data_digest s 753664 uv 3 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:33:58.769+0000 7fe2c6a64700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] do_op oi 1:41fe8a11:::smithi13140796-45:head(21'3 client.4540.0:133 dirty|data_digest s 753664 uv 3 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:33:58.769+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] do_op obc obc(1:41fe8a11:::smithi13140796-45:head rwstate(write n=1 w=0)) 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] execute_ctx 0x558e83c48a00 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] execute_ctx 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] ov 21'3 av 21'6 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] do_osd_op 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] do_osd_op setxattr _header (56) in=63b 2019-12-07T19:33:58.769+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] make_writeable 1:41fe8a11:::smithi13140796-45:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.769+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] make_writeable 1:41fe8a11:::smithi13140796-45:head done, snapset=0=[]:{} 2019-12-07T19:33:58.769+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] finish_ctx 1:41fe8a11:::smithi13140796-45:head 0x558e83c48a00 op modify 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] set mtime to 2019-12-07T19:33:58.762144+0000 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] final snapset 0=[]:{} in 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.769+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] finish_ctx object 1:41fe8a11:::smithi13140796-45:head marks clean_regions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.769+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] execute_ctx alloc reply 0x558e83b08000 result 0 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] calc_trim_to_aggressive limit = 21'2 2019-12-07T19:33:58.769+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] op order client.4540 tid 134 last was 133 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] new_repop rep_tid 5 on osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.769+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] new_repop: repgather(0x558e81123680 0'0 rep_tid=5 committed?=0 r=0) 2019-12-07T19:33:58.769+0000 7fe2c6a64700 7 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] issue_repop rep_tid 5 o 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] submit_transaction: op Op(1:41fe8a11:::smithi13140796-45:head v=21'6 tt=0'0 tid=5 reqid=client.4540.0:134 client_op=osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'3 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] get_hash_info: Getting attr on 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] get_hash_info: not in cache 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] get_hash_info: found on disk, size 188416 2019-12-07T19:33:58.770+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key = 0 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] operator(): 1:41fe8a11:::smithi13140796-45:head projected size 753664 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] start_rmw: Op(1:41fe8a11:::smithi13140796-45:head v=21'6 tt=0'0 tid=5 reqid=client.4540.0:134 client_op=osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'3 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41fe8a11:::smithi13140796-45:head=[]}) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_state_to_reads: Op(1:41fe8a11:::smithi13140796-45:head v=21'6 tt=0'0 tid=5 reqid=client.4540.0:134 client_op=osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'3 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41fe8a11:::smithi13140796-45:head=[]}) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_reads_to_commit: starting commit on Op(1:41fe8a11:::smithi13140796-45:head v=21'6 tt=0'0 tid=5 reqid=client.4540.0:134 client_op=osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'3 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41fe8a11:::smithi13140796-45:head=[]}) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] generate_transactions: 1:41fe8a11:::smithi13140796-45:head, current size is 753664 buffers are {} 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] operator(): new_size start 753664 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] operator(): to_append: {} 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] operator(): 1:41fe8a11:::smithi13140796-45:head resetting hinfo to logical size 753664 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_reads_to_commit: written: {1:41fe8a11:::smithi13140796-45:head={}} 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_reads_to_commit: op: Op(1:41fe8a11:::smithi13140796-45:head v=21'6 tt=0'0 tid=5 reqid=client.4540.0:134 client_op=osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'3 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41fe8a11:::smithi13140796-45:head=[]}) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_reads_to_commit: written_set: {1:41fe8a11:::smithi13140796-45:head=[]} 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] try_reads_to_commit: 1:41fe8a11:::smithi13140796-45:head,{} 2019-12-07T19:33:58.770+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 21/18 ECSubWrite(tid=5, reqid=client.4540.0:134, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2 -- 0x558e819db000 con 0x558e818d7c00 2019-12-07T19:33:58.770+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 21/18 ECSubWrite(tid=5, reqid=client.4540.0:134, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2 -- 0x558e83ac9000 con 0x558e818d7000 2019-12-07T19:33:58.770+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 21/18 ECSubWrite(tid=5, reqid=client.4540.0:134, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2 -- 0x558e83aca800 con 0x558e818fe400 2019-12-07T19:33:58.770+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s4 21/18 ECSubWrite(tid=5, reqid=client.4540.0:134, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2 -- 0x558e83eda000 con 0x558e818d6800 2019-12-07T19:33:58.770+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=5, reqid=client.4540.0:134, at_version=21'6, trim_to=0'0, roll_forward_to=21'3)) v2 -- 0x558e811b9800 con 0x558e818ffc00 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 earliest_dup_version = 0 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 trim 21'6 (21'3) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:134 2019-12-07T19:33:58.762144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] append_log log((0'0,21'5], crt=21'2) [21'6 (21'3) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:134 2019-12-07T19:33:58.762144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 lua=21'5 crt=21'2 lcod 21'2 mlcod 21'2 active+clean] add_log_entry 21'6 (21'3) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:134 2019-12-07T19:33:58.762144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 lua=21'5 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] rollforward: entry=21'3 (0'0) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:133 2019-12-07T19:33:58.762116+0000 0 ObjectCleanRegions clean_offsets: [753664~18446744073708797951], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 lua=21'5 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] append_log approx pg log length = 6 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 lua=21'5 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.770+0000 7fe2c6a64700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e811fac00 seq 11 2019-12-07T19:33:58.770+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 3 keys 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 3 keys = 0 2019-12-07T19:33:58.770+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.770+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fac00 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fac00 onodes 0x558e8402ea00 shared_blobs 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e8402eb08) update 0#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:41fe8a11:::smithi13140796-45:head# is 931 (485 bytes onode + 2 bytes spanning blobs + 444 bytes inline extents) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fac00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 prepare 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fac00 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 io_done 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 lua=21'5 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] op_applied version 21'6 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] eval_repop repgather(0x558e81123680 21'6 rep_tid=5 committed?=0 r=0) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e819434a0 finish 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0bce0 prio 196 cost 0 latency 0.001048 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=3, last_complete=21'4, committed=1, applied=1)) v2 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:4986b721:::smithi13140796-43:head v=21'4 tt=0'0 tid=3 reqid=client.4540.0:128 client_op=osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4986b721:::smithi13140796-43:head=[]}) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'3 crt=21'3 lcod 21'2 mlcod 21'2 active+clean] repop_all_committed: repop tid 3 all committed 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] eval_repop repgather(0x558e81123b00 21'4 rep_tid=3 committed?=1 r=0) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] commit: repgather(0x558e81123b00 21'4 rep_tid=3 committed?=1 r=0) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] log_op_stats osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 0 outb 0 lat 0.020338 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] sending reply on osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x558e811dcc80 2019-12-07T19:33:58.770+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(128 smithi13140796-43 [setxattr (56)] v21'4 uv4 ondisk = 0) v8 -- 0x558e811dcc80 con 0x558e83d68000 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.770+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] publish_stats_to_osd 21:16 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] removing repgather(0x558e81123b00 21'4 rep_tid=3 committed?=1 r=0) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] q front is repgather(0x558e81123b00 21'4 rep_tid=3 committed?=1 r=0) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] remove_repop repgather(0x558e81123b00 21'4 rep_tid=3 committed?=1 r=0) 2019-12-07T19:33:58.770+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 26 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84029400 con 0x558e818fe400 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] requeue_op 0x558e83b0a2c0 2019-12-07T19:33:58.770+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e818058c0 prio 196 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.770+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) shuffled w/ OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] try_finish_rmw: Op(1:4986b721:::smithi13140796-43:head v=21'4 tt=0'0 tid=3 reqid=client.4540.0:128 client_op=osd_op(client.4540.0:128 1.2s0 1:4986b721:::smithi13140796-43:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4986b721:::smithi13140796-43:head=[]}) 2019-12-07T19:33:58.770+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 11 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83fee400 con 0x558e818ffc00 2019-12-07T19:33:58.770+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81943600 prio 196 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.770+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.770+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0bce0 finish 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.770+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.770+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.770+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.770+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a2c0 prio 63 cost 0 latency 0.020493 osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] 2019-12-07T19:33:58.770+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] _handle_message: osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.770+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_op: op osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.770+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.770+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_op osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.770+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_object_context: found obc in cache: 0x558e83dd0900 2019-12-07T19:33:58.770+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_object_context: 0x558e83dd0900 1:4986b721:::smithi13140796-43:head rwstate(none n=0 w=0) oi: 1:4986b721:::smithi13140796-43:head(21'4 client.4540.0:128 dirty|data_digest s 720896 uv 4 dd 2bc8c113 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e811239e0 snapset: 0=[]:{} 2019-12-07T19:33:58.770+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] find_object_context 1:4986b721:::smithi13140796-43:head @head oi=1:4986b721:::smithi13140796-43:head(21'4 client.4540.0:128 dirty|data_digest s 720896 uv 4 dd 2bc8c113 alloc_hint [0 0 0]) 2019-12-07T19:33:58.770+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_op oi 1:4986b721:::smithi13140796-43:head(21'4 client.4540.0:128 dirty|data_digest s 720896 uv 4 dd 2bc8c113 alloc_hint [0 0 0]) 2019-12-07T19:33:58.770+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_op obc obc(1:4986b721:::smithi13140796-43:head rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.770+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] execute_ctx 0x558e8122b100 2019-12-07T19:33:58.770+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] execute_ctx 1:4986b721:::smithi13140796-43:head [read 0~1] ov 21'4 2019-12-07T19:33:58.770+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_osd_op 1:4986b721:::smithi13140796-43:head [read 0~1] 2019-12-07T19:33:58.770+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_osd_op read 0~1 2019-12-07T19:33:58.770+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_read 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] async_read noted for 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_all_avail_shards: checking acting 0(0) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_all_avail_shards: checking acting 4(1) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_all_avail_shards: checking acting 5(4) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] get_all_avail_shards: checking acting 6(3) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] start_read_op: starting ReadOp(tid=6, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_read_op: starting read ReadOp(tid=6, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 21 send_incremental_map 0 -> 21 to 0x558e802a5400 v2:172.21.15.131:6822/37079 2019-12-07T19:33:58.771+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3a80e826:::osdmap.21:0# 0x0~0 2019-12-07T19:33:58.771+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~140c size 0x140c (5132) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:58.771+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8b60 blob([0x35000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e811d0230 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:58.771+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d03f0 blob([!~1000,0x36000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x558e811d0620 sbid 0x0)) need 0x1000~40c cache has 0x[1000~40c] 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3a80e826:::osdmap.21:0# 0x0~140c = 5132 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 21 add_map_bl 21 5132 bytes 2019-12-07T19:33:58.771+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- osd_map(21..21 src has 1..21) v4 -- 0x558e83dd0fc0 con 0x558e802a5400 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 21 maybe_share_map con 0x558e802a5400 v2:172.21.15.131:6822/37079 map epoch 0 -> 21 (shared) 2019-12-07T19:33:58.771+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 11 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e811f7200 con 0x558e818d6800 2019-12-07T19:33:58.771+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402ec80 con 0x558e802a5400 2019-12-07T19:33:58.771+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0b1e0 prio 196 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.771+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.2s2 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402e500 con 0x558e818d7c00 2019-12-07T19:33:58.771+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558e8402ec80 con 0x558e802a5400 2019-12-07T19:33:58.771+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.2s1 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402e780 con 0x558e818fe400 2019-12-07T19:33:58.771+0000 7fe2d427f700 15 osd.0 21 enqueue_op 0x558e81942420 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.771+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402ef00 con 0x558e818ffc00 2019-12-07T19:33:58.771+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== osd_map(21..21 src has 1..21) v4 ==== 0+0+0 (unknown 0 0 0) 0x558e83dd0fc0 con 0x558e802a5400 2019-12-07T19:33:58.771+0000 7fe2d4a80700 20 osd.0 21 OSD::ms_dispatch: osd_map(21..21 src has 1..21) v4 2019-12-07T19:33:58.771+0000 7fe2d4a80700 10 osd.0 21 do_waiters -- start 2019-12-07T19:33:58.771+0000 7fe2d4a80700 10 osd.0 21 do_waiters -- finish 2019-12-07T19:33:58.771+0000 7fe2d4a80700 20 osd.0 21 _dispatch 0x558e83dd0fc0 osd_map(21..21 src has 1..21) v4 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] do_read_op: started ReadOp(tid=6, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:4986b721:::smithi13140796-43:head,1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,6(3)=1:4986b721:::smithi13140796-43:head}, in_progress=0(0),1(2),4(1),6(3)) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a2c0 finish 2019-12-07T19:33:58.771+0000 7fe2d4a80700 3 osd.0 21 handle_osd_map epochs [21,21], i have 21, src has [1,21] 2019-12-07T19:33:58.771+0000 7fe2d4a80700 10 osd.0 21 no new maps here, dropping 2019-12-07T19:33:58.771+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0be40 prio 196 cost 0 latency 0.001706 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=4, last_complete=21'5, committed=1, applied=1)) v2 2019-12-07T19:33:58.771+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 72 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818eee00 con 0x558e818d7c00 2019-12-07T19:33:58.771+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0b760 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:41df217a:::smithi13140796-39:head v=21'5 tt=0'0 tid=4 reqid=client.4540.0:116 client_op=osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:33:58.771+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'4 crt=21'3 lcod 21'3 mlcod 21'3 active+clean] repop_all_committed: repop tid 4 all committed 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] eval_repop repgather(0x558e811238c0 21'5 rep_tid=4 committed?=1 r=0) 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] commit: repgather(0x558e811238c0 21'5 rep_tid=4 committed?=1 r=0) 2019-12-07T19:33:58.771+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] log_op_stats osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 0 outb 0 lat 0.022532 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] sending reply on osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x558e83b09680 2019-12-07T19:33:58.771+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(116 smithi13140796-39 [setxattr (56)] v21'5 uv5 ondisk = 0) v8 -- 0x558e83b09680 con 0x558e83d68000 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.771+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] publish_stats_to_osd 21:17 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] removing repgather(0x558e811238c0 21'5 rep_tid=4 committed?=1 r=0) 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] q front is repgather(0x558e811238c0 21'5 rep_tid=4 committed?=1 r=0) 2019-12-07T19:33:58.771+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] remove_repop repgather(0x558e811238c0 21'5 rep_tid=4 committed?=1 r=0) 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] requeue_op 0x558e819438c0 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) shuffled w/ OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] try_finish_rmw: Op(1:41df217a:::smithi13140796-39:head v=21'5 tt=0'0 tid=4 reqid=client.4540.0:116 client_op=osd_op(client.4540.0:116 1.2s0 1:41df217a:::smithi13140796-39:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'2 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0be40 finish 2019-12-07T19:33:58.771+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e819438c0 prio 63 cost 0 latency 0.022613 osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.771+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op: op osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.771+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_object_context: found obc in cache: 0x558e81222900 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_object_context: 0x558e81222900 1:41df217a:::smithi13140796-39:head rwstate(none n=0 w=0) oi: 1:41df217a:::smithi13140796-39:head(21'5 client.4540.0:116 dirty|data_digest s 638976 uv 5 dd b038798b alloc_hint [0 0 0]) exists: 1 ssc: 0x558e811237a0 snapset: 0=[]:{} 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] find_object_context 1:41df217a:::smithi13140796-39:head @head oi=1:41df217a:::smithi13140796-39:head(21'5 client.4540.0:116 dirty|data_digest s 638976 uv 5 dd b038798b alloc_hint [0 0 0]) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op oi 1:41df217a:::smithi13140796-39:head(21'5 client.4540.0:116 dirty|data_digest s 638976 uv 5 dd b038798b alloc_hint [0 0 0]) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op obc obc(1:41df217a:::smithi13140796-39:head rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx 0x558e83c46000 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx 1:41df217a:::smithi13140796-39:head [read 0~1] ov 21'5 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op 1:41df217a:::smithi13140796-39:head [read 0~1] 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op read 0~1 2019-12-07T19:33:58.771+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_read 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] async_read noted for 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_all_avail_shards: checking acting 0(0) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_all_avail_shards: checking acting 4(1) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_all_avail_shards: checking acting 5(4) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_all_avail_shards: checking acting 6(3) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] start_read_op: starting ReadOp(tid=7, to_read={1:41df217a:::smithi13140796-39:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:41df217a:::smithi13140796-39:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_read_op: starting read ReadOp(tid=7, to_read={1:41df217a:::smithi13140796-39:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:41df217a:::smithi13140796-39:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402f180 con 0x558e802a5400 2019-12-07T19:33:58.771+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.2s2 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e83cb3b80 con 0x558e818d7c00 2019-12-07T19:33:58.771+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.2s1 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e81a03900 con 0x558e818fe400 2019-12-07T19:33:58.771+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558e8402f180 con 0x558e802a5400 2019-12-07T19:33:58.771+0000 7fe2d427f700 15 osd.0 21 enqueue_op 0x558e84014000 prio 127 cost 0 latency 0.000017 epoch 21 MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.771+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e83c72c80 con 0x558e818ffc00 2019-12-07T19:33:58.771+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_read_op: started ReadOp(tid=7, to_read={1:41df217a:::smithi13140796-39:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:41df217a:::smithi13140796-39:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={1:41df217a:::smithi13140796-39:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:41df217a:::smithi13140796-39:head,1(2)=1:41df217a:::smithi13140796-39:head,4(1)=1:41df217a:::smithi13140796-39:head,6(3)=1:41df217a:::smithi13140796-39:head}, in_progress=0(0),1(2),4(1),6(3)) 2019-12-07T19:33:58.771+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e819438c0 finish 2019-12-07T19:33:58.771+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 27 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 ==== 4261+0+0 (unknown 0 0 0) 0x558e83b09680 con 0x558e818fe400 2019-12-07T19:33:58.771+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.771+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943e40 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.771+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b1e0 prio 196 cost 0 latency 0.000439 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b1e0 finish 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943600 prio 196 cost 0 latency 0.000898 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943600 finish 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b760 prio 196 cost 0 latency 0.000407 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b760 finish 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.771+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 12 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 ==== 4261+0+0 (unknown 0 0 0) 0x558e83cb3b80 con 0x558e818ffc00 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e818058c0 prio 196 cost 0 latency 0.001005 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.771+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0b600 prio 127 cost 0 latency 0.000013 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.771+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e818058c0 finish 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81942420 prio 127 cost 0 latency 0.000572 MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=6, to_read={1:4986b721:::smithi13140796-43:head=0,4096,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.771+0000 7fe2c6a64700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.771+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 0x0~1000 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x2c000 (180224) 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff2850 blob([0x1ea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff27e0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff2850 blob([0x1ea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff27e0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff2850 blob([0x1ea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff27e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83ff27f8 in 0x558e802195e0) _discard 0x0~1000 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 _add level 1 near 0 on buffer(0x558e81171200 space 0x558e83ff27f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 0x0~1000 = 4096 2019-12-07T19:33:58.771+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.771+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_hash_info: Getting attr on 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.772+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_hash_info: not in cache 1:4986b721:::smithi13140796-43:head 2019-12-07T19:33:58.772+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:33:58.772+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_hash_info: found on disk, size 180224 2019-12-07T19:33:58.772+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key 2019-12-07T19:33:58.772+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key = 0 2019-12-07T19:33:58.772+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 73 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 ==== 4261+0+0 (unknown 0 0 0) 0x558e83c72c80 con 0x558e818d7c00 2019-12-07T19:33:58.772+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0a000 prio 127 cost 0 latency 0.000014 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.772+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.772+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 -- 0x558e83c72f00 con 0x558e802a5400 2019-12-07T19:33:58.772+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81942420 finish 2019-12-07T19:33:58.772+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558e83c72f00 con 0x558e802a5400 2019-12-07T19:33:58.772+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2d427f700 15 osd.0 21 enqueue_op 0x558e84014160 prio 127 cost 0 latency 0.000009 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.772+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.772+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.772+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 13 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 ==== 4261+0+0 (unknown 0 0 0) 0x558e8402ef00 con 0x558e818ffc00 2019-12-07T19:33:58.772+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0b4a0 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.772+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.772+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.772+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 28 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 ==== 4261+0+0 (unknown 0 0 0) 0x558e81a03900 con 0x558e818fe400 2019-12-07T19:33:58.772+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943a20 prio 127 cost 0 latency 0.000013 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.772+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.772+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943e40 prio 127 cost 0 latency 0.000482 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.772+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.772+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=6, attrs_read=0) 2019-12-07T19:33:58.772+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=6, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(1),4096]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:4986b721:::smithi13140796-43:head,1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,6(3)=1:4986b721:::smithi13140796-43:head}, in_progress=0(0),1(2),6(3)) 2019-12-07T19:33:58.772+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943e40 finish 2019-12-07T19:33:58.772+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b600 prio 127 cost 0 latency 0.000420 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=6, attrs_read=0) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=6, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(1),4096, 6(3),4096]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:4986b721:::smithi13140796-43:head,1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,6(3)=1:4986b721:::smithi13140796-43:head}, in_progress=0(0),1(2)) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b600 finish 2019-12-07T19:33:58.772+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 74 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 ==== 4261+0+0 (unknown 0 0 0) 0x558e8402e500 con 0x558e818d7c00 2019-12-07T19:33:58.772+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0a160 prio 127 cost 0 latency 0.000012 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e84014000 prio 127 cost 0 latency 0.000648 MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=7, to_read={1:41df217a:::smithi13140796-39:head=0,4096,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.772+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.772+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 0x0~1000 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x27000 (159744) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd2ee0 blob([0x1c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e70 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd2ee0 blob([0x1c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e70 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd2ee0 blob([0x1c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e70 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd2e88 in 0x558e802195e0) _discard 0x0~1000 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83d114d0 space 0x558e83dd2e88 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 0x0~1000 = 4096 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_hash_info: Getting attr on 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_hash_info: not in cache 1:41df217a:::smithi13140796-39:head 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_hash_info: found on disk, size 159744 2019-12-07T19:33:58.772+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key = 0 2019-12-07T19:33:58.772+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 -- 0x558e83cb3b80 con 0x558e802a5400 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e84014000 finish 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.772+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558e83cb3b80 con 0x558e802a5400 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.772+0000 7fe2d427f700 15 osd.0 21 enqueue_op 0x558e83b0a6e0 prio 127 cost 0 latency 0.000014 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.772+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a000 prio 127 cost 0 latency 0.000453 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=6, attrs_read=0) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=6, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(2),4096, 4(1),4096, 6(3),4096]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:4986b721:::smithi13140796-43:head,1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,6(3)=1:4986b721:::smithi13140796-43:head}, in_progress=0(0)) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a000 finish 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81943a20 prio 127 cost 0 latency 0.000446 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=7, attrs_read=0) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=7, to_read={1:41df217a:::smithi13140796-39:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:41df217a:::smithi13140796-39:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(1),4096]))}, priority=127, obj_to_source={1:41df217a:::smithi13140796-39:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:41df217a:::smithi13140796-39:head,1(2)=1:41df217a:::smithi13140796-39:head,4(1)=1:41df217a:::smithi13140796-39:head,6(3)=1:41df217a:::smithi13140796-39:head}, in_progress=0(0),1(2),6(3)) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81943a20 finish 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b4a0 prio 127 cost 0 latency 0.000519 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=7, attrs_read=0) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=7, to_read={1:41df217a:::smithi13140796-39:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:41df217a:::smithi13140796-39:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [4(1),4096, 6(3),4096]))}, priority=127, obj_to_source={1:41df217a:::smithi13140796-39:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:41df217a:::smithi13140796-39:head,1(2)=1:41df217a:::smithi13140796-39:head,4(1)=1:41df217a:::smithi13140796-39:head,6(3)=1:41df217a:::smithi13140796-39:head}, in_progress=0(0),1(2)) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b4a0 finish 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e84014160 prio 127 cost 0 latency 0.000563 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=6, attrs_read=0)) v2 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=6, attrs_read=0) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply have shard=0 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply have shard=2 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply have shard=1 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply have shard=3 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply Complete: ReadOp(tid=6, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [0(0),4096, 1(2),4096, 4(1),4096, 6(3),4096]))}, priority=127, obj_to_source={1:4986b721:::smithi13140796-43:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:4986b721:::smithi13140796-43:head,1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,6(3)=1:4986b721:::smithi13140796-43:head}, in_progress=) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] objects_read_async_cb: got: {1:4986b721:::smithi13140796-43:head=0,{0~16384(16384)}} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx 0x558e8122b100 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx 1:4986b721:::smithi13140796-43:head [read 0~1 out=1b] ov 21'4 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op 1:4986b721:::smithi13140796-43:head [read 0~1 out=1b] 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op read 0~1 out=1b 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx alloc reply 0x558e8402f180 result 0 2019-12-07T19:33:58.772+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:33:58.772+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] log_op_stats osd_op(client.4540.0:129 1.2s0 1:4986b721:::smithi13140796-43:head [read 0~1 out=1b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 inb 0 outb 1 lat 0.022462 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.772+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] publish_stats_to_osd 21:18 2019-12-07T19:33:58.772+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(129 smithi13140796-43 [read 0~1 out=1b] v0'0 uv4 ondisk = 0) v8 -- 0x558e8402f180 con 0x558e83d68000 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e84014160 finish 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a160 prio 127 cost 0 latency 0.000680 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=7, attrs_read=0) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=7, to_read={1:41df217a:::smithi13140796-39:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:41df217a:::smithi13140796-39:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [1(2),4096, 4(1),4096, 6(3),4096]))}, priority=127, obj_to_source={1:41df217a:::smithi13140796-39:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:41df217a:::smithi13140796-39:head,1(2)=1:41df217a:::smithi13140796-39:head,4(1)=1:41df217a:::smithi13140796-39:head,6(3)=1:41df217a:::smithi13140796-39:head}, in_progress=0(0)) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a160 finish 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a6e0 prio 127 cost 0 latency 0.000484 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=7, attrs_read=0)) v2 2019-12-07T19:33:58.772+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=7, attrs_read=0) 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply have shard=0 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply have shard=2 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply have shard=1 2019-12-07T19:33:58.772+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply have shard=3 2019-12-07T19:33:58.773+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] handle_sub_read_reply Complete: ReadOp(tid=7, to_read={1:41df217a:::smithi13140796-39:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:41df217a:::smithi13140796-39:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [0(0),4096, 1(2),4096, 4(1),4096, 6(3),4096]))}, priority=127, obj_to_source={1:41df217a:::smithi13140796-39:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:41df217a:::smithi13140796-39:head,1(2)=1:41df217a:::smithi13140796-39:head,4(1)=1:41df217a:::smithi13140796-39:head,6(3)=1:41df217a:::smithi13140796-39:head}, in_progress=) 2019-12-07T19:33:58.773+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] objects_read_async_cb: got: {1:41df217a:::smithi13140796-39:head=0,{0~16384(16384)}} 2019-12-07T19:33:58.773+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:33:58.773+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx 0x558e83c46000 2019-12-07T19:33:58.773+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx 1:41df217a:::smithi13140796-39:head [read 0~1 out=1b] ov 21'5 2019-12-07T19:33:58.773+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op 1:41df217a:::smithi13140796-39:head [read 0~1 out=1b] 2019-12-07T19:33:58.773+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op read 0~1 out=1b 2019-12-07T19:33:58.773+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx alloc reply 0x558e8402e500 result 0 2019-12-07T19:33:58.773+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:33:58.773+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] log_op_stats osd_op(client.4540.0:117 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~1 out=1b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 inb 0 outb 1 lat 0.024239 2019-12-07T19:33:58.773+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.773+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] publish_stats_to_osd 21:19 2019-12-07T19:33:58.773+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(117 smithi13140796-39 [read 0~1 out=1b] v0'0 uv5 ondisk = 0) v8 -- 0x558e8402e500 con 0x558e83d68000 2019-12-07T19:33:58.773+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a6e0 finish 2019-12-07T19:33:58.773+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.773+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 16 ==== MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:136, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 194484+0+0 (unknown 0 0 0) 0x558e83cbf000 con 0x558e818d6000 2019-12-07T19:33:58.773+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0a580 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:136, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.773+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:136, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.773+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.773+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.773+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81942dc0 prio 63 cost 0 latency 0.011765 osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.773+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.773+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op: op osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.773+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.773+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.773+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_object_context: found obc in cache: 0x558e83dd1200 2019-12-07T19:33:58.773+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_object_context: 0x558e83dd1200 1:41fe8a11:::smithi13140796-45:head rwstate(write n=1 w=0) oi: 1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e81123c20 snapset: 0=[]:{} 2019-12-07T19:33:58.773+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] find_object_context 1:41fe8a11:::smithi13140796-45:head @head oi=1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:33:58.773+0000 7fe2c6a64700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op oi 1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:33:58.773+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.773+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81942dc0 finish 2019-12-07T19:33:58.773+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.774+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.774+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.774+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1971 < min_flush_size 524288 2019-12-07T19:33:58.774+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ea00 had 1 2019-12-07T19:33:58.774+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:33:58.774+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1994 < min_flush_size 524288 2019-12-07T19:33:58.774+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1994 < min_flush_size 524288 2019-12-07T19:33:58.774+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.774+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3be26~7ca to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.774+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3be26 0x3be26~7ca to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.774+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.774+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xbe26 2019-12-07T19:33:58.774+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xe26 2019-12-07T19:33:58.774+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.774+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x5f0 and padding block with 0xa10 2019-12-07T19:33:58.774+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3c5f0 2019-12-07T19:33:58.774+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.774+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.774+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.774+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.774+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000117505s (0.000000189s flush + 0.000117316s kv commit) 2019-12-07T19:33:58.774+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fac00> 2019-12-07T19:33:58.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.774+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 kv_submitted 2019-12-07T19:33:58.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fac00 2019-12-07T19:33:58.774+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 finishing 2019-12-07T19:33:58.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fac00 onodes 0x558e8402ea00 2019-12-07T19:33:58.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fac00 done 2019-12-07T19:33:58.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:33:58.774+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fac00 [] 2019-12-07T19:33:58.774+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.774+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.774+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:136, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:136, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558e819e7000 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 osd.0 21 dequeue_op 0x558e83b0a580 prio 127 cost 0 latency 0.002852 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:136, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.ds5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=24, reqid=client.4540.0:136, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'4], crt=21'4) [21'5 (0'0) modify 1:be42858c:::smithi13140796-46:head by client.4540.0:136 2019-12-07T19:33:58.768176+0000 0 ObjectCleanRegions clean_offsets: [770048~18446744073708781567], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] add_log_entry 21'5 (0'0) modify 1:be42858c:::smithi13140796-46:head by client.4540.0:136 2019-12-07T19:33:58.768176+0000 0 ObjectCleanRegions clean_offsets: [770048~18446744073708781567], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.776+0000 7fe2c9269700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e83d4d200 seq 12 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.ds5_head 0x558e817b5860) get_onode oid 5#1:be42858c:::smithi13140796-46:head# key 0x858000000000000001be42858c21736d'ithi13140796-46!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.776+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.ds5_head 5#1:be42858c:::smithi13140796-46:head# 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1103 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.ds5_head 5#1:be42858c:::smithi13140796-46:head# = 0 2019-12-07T19:33:58.776+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.ds5_head 5#1:be42858c:::smithi13140796-46:head# 2 keys 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.ds5_head 5#1:be42858c:::smithi13140796-46:head# 2 keys = 0 2019-12-07T19:33:58.776+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.ds5_head 5#1:be42858c:::smithi13140796-46:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.ds5_head 5#1:be42858c:::smithi13140796-46:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.776+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.ds5_head 5#1:be42858c:::smithi13140796-46:head# 0x0~2f000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#1:be42858c:::smithi13140796-46:head# 0x0~2f000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~2f000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83d4d200 24 blobs 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 fbmap_alloc 0x558e80233300 allocate 0x2f000/1000,2f000,0 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x244000~2f000/1000,2f000,0 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x244000~2f000] 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83edf8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194f0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83edf8f0 blob([0x244000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f0a0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edf8f0) get_ref 0x0~2000 Blob(0x558e83edf8f0 blob([0x244000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f0a0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edf8f0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83edf8f0 blob([0x244000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f0a0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e8194f0b8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d02af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d02d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d02af0 blob([0x246000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02d20 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83d02af0) get_ref 0x0~2000 Blob(0x558e83d02af0 blob([0x246000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02d20 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83d02af0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83d02af0 blob([0x246000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02d20 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83d02d38 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d02ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d02f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d02ee0 blob([0x248000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02f50 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83d02ee0) get_ref 0x0~2000 Blob(0x558e83d02ee0 blob([0x248000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02f50 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83d02ee0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83d02ee0 blob([0x248000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02f50 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83d02f68 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d02fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d02fc0 blob([0x24a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03030 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83d02fc0) get_ref 0x0~2000 Blob(0x558e83d02fc0 blob([0x24a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03030 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83d02fc0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83d02fc0 blob([0x24a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03030 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83d03048 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d030a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d030a0 blob([0x24c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03110 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83d030a0) get_ref 0x0~2000 Blob(0x558e83d030a0 blob([0x24c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03110 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83d030a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83d030a0 blob([0x24c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03110 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83d03128 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d02d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac47e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d02d90 blob([0x24e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac47e0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83d02d90) get_ref 0x0~2000 Blob(0x558e83d02d90 blob([0x24e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac47e0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83d02d90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83d02d90 blob([0x24e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac47e0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83ac47f8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac4700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac4700 blob([0x250000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5dc0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83ac4700) get_ref 0x0~2000 Blob(0x558e83ac4700 blob([0x250000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5dc0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83ac4700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83ac4700 blob([0x250000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5dc0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83ac5dd8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd2540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edf340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd2540 blob([0x252000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf340 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83dd2540) get_ref 0x0~2000 Blob(0x558e83dd2540 blob([0x252000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf340 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83dd2540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83dd2540 blob([0x252000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edf340 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83edf358 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83edf5e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edf570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83edf5e0 blob([0x254000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf570 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edf5e0) get_ref 0x0~2000 Blob(0x558e83edf5e0 blob([0x254000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf570 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edf5e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83edf5e0 blob([0x254000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edf570 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83edf588 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83edfe30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edfdc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83edfe30 blob([0x256000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edfdc0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edfe30) get_ref 0x0~2000 Blob(0x558e83edfe30 blob([0x256000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edfdc0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edfe30) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83edfe30 blob([0x256000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edfdc0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83edfdd8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83edfd50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edfce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83edfd50 blob([0x258000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edfce0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edfd50) get_ref 0x0~2000 Blob(0x558e83edfd50 blob([0x258000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edfce0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edfd50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83edfd50 blob([0x258000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edfce0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83edfcf8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83edfc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edfc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83edfc70 blob([0x25a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edfc00 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edfc70) get_ref 0x0~2000 Blob(0x558e83edfc70 blob([0x25a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edfc00 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edfc70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83edfc70 blob([0x25a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edfc00 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83edfc18 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83edfb90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edfb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83edfb90 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edfb20 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edfb90) get_ref 0x0~2000 Blob(0x558e83edfb90 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edfb20 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edfb90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83edfb90 blob([0x25c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edfb20 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83edfb38 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83edf490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edf420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83edf490 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf420 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edf490) get_ref 0x0~2000 Blob(0x558e83edf490 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf420 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edf490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83edf490 blob([0x25e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edf420 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83edf438 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83edff10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edff80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83edff10 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edff80 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edff10) get_ref 0x0~2000 Blob(0x558e83edff10 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edff80 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83edff10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e83edff10 blob([0x260000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edff80 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83edff98 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6e000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6e000 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e070 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e000) get_ref 0x0~2000 Blob(0x558e83e6e000 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e070 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e83e6e000 blob([0x262000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6e070 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83e6e088 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6e0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6e0e0 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e150 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e0e0) get_ref 0x0~2000 Blob(0x558e83e6e0e0 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e150 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e0e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e83e6e0e0 blob([0x264000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6e150 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83e6e168 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6e1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6e1c0 blob([0x266000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e230 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e1c0) get_ref 0x0~2000 Blob(0x558e83e6e1c0 blob([0x266000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e230 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e1c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e83e6e1c0 blob([0x266000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6e230 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83e6e248 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6e2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6e2a0 blob([0x268000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e310 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e2a0) get_ref 0x0~2000 Blob(0x558e83e6e2a0 blob([0x268000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e310 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e2a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e83e6e2a0 blob([0x268000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6e310 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83e6e328 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6e380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6e380 blob([0x26a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e3f0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e380) get_ref 0x0~2000 Blob(0x558e83e6e380 blob([0x26a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e3f0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e380) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e83e6e380 blob([0x26a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6e3f0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83e6e408 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6e460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6e460 blob([0x26c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e4d0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e460) get_ref 0x0~2000 Blob(0x558e83e6e460 blob([0x26c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e4d0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e83e6e460 blob([0x26c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6e4d0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83e6e4e8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6e540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6e540 blob([0x26e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e5b0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e540) get_ref 0x0~2000 Blob(0x558e83e6e540 blob([0x26e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e5b0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558e83e6e540 blob([0x26e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6e5b0 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83e6e5c8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6e620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6e620 blob([0x270000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e690 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e620) get_ref 0x0~2000 Blob(0x558e83e6e620 blob([0x270000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e690 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558e83e6e620 blob([0x270000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6e690 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83e6e6a8 in 0x558e80219dc0) _discard 0x0~2000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6e700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6e700 blob([0x272000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e770 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e700) get_ref 0x0~1000 Blob(0x558e83e6e700 blob([0x272000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6e770 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6e700) get_ref init 0x1000, 1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2e000~1000: 0x0~1000 Blob(0x558e83e6e700 blob([0x272000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83e6e770 sbid 0x0)) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83e6e788 in 0x558e80219dc0) _discard 0x0~1000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x2f000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.extentmap(0x558e83cb3288) dirty_range mark inline shard dirty 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.ds5_head 5#1:be42858c:::smithi13140796-46:head# 0x0~2f000 = 0 2019-12-07T19:33:58.776+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.ds5_head 5#1:be42858c:::smithi13140796-46:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.ds5_head 5#1:be42858c:::smithi13140796-46:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.776+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4d200 cost 294134 (25 ios * 4000 + 194134 bytes) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4d200 onodes 0x558e83cb3180 shared_blobs 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.extentmap(0x558e83cb3288) update 5#1:be42858c:::smithi13140796-46:head# 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore.extentmap(0x558e83cb3288) update inline shard 456 bytes from 24 extents 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:be42858c:::smithi13140796-46:head# is 875 (413 bytes onode + 2 bytes spanning blobs + 460 bytes inline extents) 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4d200 allocated 0x[244000~2f000] released 0x[] 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 freelist allocate 0x244000~2f000 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 freelist _xor first_key 0x200000 last_key 0x200000 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d200 prepare 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83d4d200 2019-12-07T19:33:58.776+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 10 ==== osd_op(client.4540.0:142 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 264+0+425984 (crc 0 0 0) 0x558e83aaeb00 con 0x558e83d68000 2019-12-07T19:33:58.776+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e84014000 prio 63 cost 425984 latency 0.000018 epoch 21 osd_op(client.4540.0:142 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.776+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:142 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 425984 e21) 2019-12-07T19:33:58.776+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.776+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:142 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 425984 e21) queued 2019-12-07T19:33:58.776+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.776+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:142 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 425984 e21) pg 0x558e81075000 2019-12-07T19:33:58.776+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e84014000 prio 63 cost 425984 latency 0.000060 osd_op(client.4540.0:142 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.776+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] _handle_message: osd_op(client.4540.0:142 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.776+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op: op osd_op(client.4540.0:142 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~425984 in=425984b,stat] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.776+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.776+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op osd_op(client.4540.0:142 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~425984 in=425984b,stat] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write may_read -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.776+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_object_context: obc NOT found in cache: 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.776+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# _ 2019-12-07T19:33:58.776+0000 7fe2c9269700 10 osd.0 21 dequeue_op 0x558e83b0a580 finish 2019-12-07T19:33:58.776+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.776+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:4c9e444c:::smithi13140796-48:head# key 0x8080000000000000014c9e44'L!smithi13140796-48!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.776+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:33:58.776+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# _ = -2 2019-12-07T19:33:58.776+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_object_context: no obc for soid 1:4c9e444c:::smithi13140796-48:head but can_create 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] create_object_context 0x558e83ff0000 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] populate_obc_watchers 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_object_context: 0x558e83ff0000 1:4c9e444c:::smithi13140796-48:head rwstate(none n=0 w=0) oi: 1:4c9e444c:::smithi13140796-48:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) ssc: 0x558e81123d40 snapset: 0=[]:{} 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] find_object_context 1:4c9e444c:::smithi13140796-48:head @head oi=1:4c9e444c:::smithi13140796-48:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.777+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op oi 1:4c9e444c:::smithi13140796-48:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_op obc obc(1:4c9e444c:::smithi13140796-48:head(dne) rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx 0x558e83c46000 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx 1:4c9e444c:::smithi13140796-48:head [append 0~425984 in=425984b,stat] ov 0'0 av 21'7 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op 1:4c9e444c:::smithi13140796-48:head [append 0~425984 in=425984b,stat] 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op append 0~425984 in=425984b 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op 1:4c9e444c:::smithi13140796-48:head [write 0~425984 in=425984b] 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op write 0~425984 in=425984b 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] clean_regions modifiedclean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] do_osd_op stat 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] stat oi has 425984 0.000000 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] make_writeable 1:4c9e444c:::smithi13140796-48:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] setting DIRTY flag 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] make_writeable 1:4c9e444c:::smithi13140796-48:head done, snapset=0=[]:{} 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] finish_ctx 1:4c9e444c:::smithi13140796-48:head 0x558e83c46000 op modify 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] set mtime to 2019-12-07T19:33:58.777249+0000 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] final snapset 0=[]:{} in 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] finish_ctx object 1:4c9e444c:::smithi13140796-48:head marks clean_regions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] execute_ctx alloc reply 0x558e83cb3b80 result 0 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] calc_trim_to_aggressive limit = 21'3 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] op order client.4540 tid 142 (first) 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] new_repop rep_tid 8 on osd_op(client.4540.0:142 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~425984 in=425984b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] new_repop: repgather(0x558e81123e60 0'0 rep_tid=8 committed?=0 r=0) 2019-12-07T19:33:58.777+0000 7fe2caa6c700 7 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] issue_repop rep_tid 8 o 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] submit_transaction: op Op(1:4c9e444c:::smithi13140796-48:head v=21'7 tt=0'0 tid=8 reqid=client.4540.0:142 client_op=osd_op(client.4540.0:142 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~425984 in=425984b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'5 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_hash_info: Getting attr on 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] get_hash_info: not in cache 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:4c9e444c:::smithi13140796-48:head# key 0x8080000000000000014c9e44'L!smithi13140796-48!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] operator(): 1:4c9e444c:::smithi13140796-48:head projected size 425984 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] start_rmw: Op(1:4c9e444c:::smithi13140796-48:head v=21'7 tt=0'0 tid=8 reqid=client.4540.0:142 client_op=osd_op(client.4540.0:142 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~425984 in=425984b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'5 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[0~425984]}) 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] try_state_to_reads: Op(1:4c9e444c:::smithi13140796-48:head v=21'7 tt=0'0 tid=8 reqid=client.4540.0:142 client_op=osd_op(client.4540.0:142 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~425984 in=425984b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'5 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[0~425984]}) 2019-12-07T19:33:58.777+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] try_reads_to_commit: starting commit on Op(1:4c9e444c:::smithi13140796-48:head v=21'7 tt=0'0 tid=8 reqid=client.4540.0:142 client_op=osd_op(client.4540.0:142 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~425984 in=425984b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'5 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[0~425984]}) 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] try_reads_to_commit: ExtentCache( Extents(1:4c9e444c:::smithi13140796-48:head)[ Extent(0~425984:7) ) 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] generate_transactions: 1:4c9e444c:::smithi13140796-48:head, current size is 0 buffers are {0~425984(425984)} 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] operator(): new_size start 0 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] operator(): adding buffer_update 0,425984 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] operator(): to_append: {0~425984(425984)} 2019-12-07T19:33:58.777+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] operator(): appending 0~425984 2019-12-07T19:33:58.777+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d200 aio_wait 2019-12-07T19:33:58.777+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4d200 2019-12-07T19:33:58.777+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d200 io_done 2019-12-07T19:33:58.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.777+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1587 < min_flush_size 524288 2019-12-07T19:33:58.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3180 had 1 2019-12-07T19:33:58.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:33:58.777+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1610 < min_flush_size 524288 2019-12-07T19:33:58.777+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1610 < min_flush_size 524288 2019-12-07T19:33:58.777+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.777+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3c5f0~64a to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.777+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3c5f0 0x3c5f0~64a to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.777+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.777+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xc5f0 2019-12-07T19:33:58.777+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x5f0 2019-12-07T19:33:58.777+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.777+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc3a and padding block with 0x3c6 2019-12-07T19:33:58.777+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3cc3a 2019-12-07T19:33:58.777+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.777+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.777+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.777+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000158988s (0.000048048s flush + 0.000110940s kv commit) 2019-12-07T19:33:58.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4d200> 2019-12-07T19:33:58.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.777+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d200 kv_submitted 2019-12-07T19:33:58.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4d200 2019-12-07T19:33:58.777+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d200 finishing 2019-12-07T19:33:58.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4d200 onodes 0x558e83cb3180 2019-12-07T19:33:58.777+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194f0b8 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d11170 space 0x558e8194f0b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac47f8 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8fc20 space 0x558e83ac47f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.777+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac5dd8 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8e990 space 0x558e83ac5dd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d02d38 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d11200 space 0x558e83d02d38 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d02f68 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d11290 space 0x558e83d02f68 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03048 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d11b00 space 0x558e83d03048 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03128 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8eab0 space 0x558e83d03128 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6e088 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10510 space 0x558e83e6e088 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6e168 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10360 space 0x558e83e6e168 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6e248 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d11320 space 0x558e83e6e248 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6e328 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10990 space 0x558e83e6e328 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6e408 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d10900 space 0x558e83e6e408 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6e4e8 in 0x558e80219dc0) _finish_write discard buffer(0x558e84032090 space 0x558e83e6e4e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6e5c8 in 0x558e80219dc0) _finish_write discard buffer(0x558e84032120 space 0x558e83e6e5c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6e6a8 in 0x558e80219dc0) _finish_write discard buffer(0x558e840321b0 space 0x558e83e6e6a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6e788 in 0x558e80219dc0) _finish_write discard buffer(0x558e84032240 space 0x558e83e6e788 0x0~1000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edf358 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8ecf0 space 0x558e83edf358 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edf438 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d105a0 space 0x558e83edf438 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edf588 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8f170 space 0x558e83edf588 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edfb38 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d11a70 space 0x558e83edfb38 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edfc18 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d119e0 space 0x558e83edfc18 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=24, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558e8118f400 con 0x558e818d6000 2019-12-07T19:33:58.778+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edfcf8 in 0x558e80219dc0) _finish_write discard buffer(0x558e81171050 space 0x558e83edfcf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edfdd8 in 0x558e80219dc0) _finish_write discard buffer(0x558e83c8f290 space 0x558e83edfdd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edff98 in 0x558e80219dc0) _finish_write discard buffer(0x558e83d103f0 space 0x558e83edff98 0x0~2000 writing nocache) 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4d200 done 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:33:58.778+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4d200 [] 2019-12-07T19:33:58.778+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.778+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.778+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 39 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83fef400 con 0x558e818d7000 2019-12-07T19:33:58.778+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e840142c0 prio 196 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.778+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.778+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.778+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.778+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] encode_and_write: 1:4c9e444c:::smithi13140796-48:head new_size 425984 2019-12-07T19:33:58.778+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] operator(): 1:4c9e444c:::smithi13140796-48:head resetting hinfo to logical size 425984 2019-12-07T19:33:58.778+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] operator(): marking append 0 2019-12-07T19:33:58.778+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] try_reads_to_commit: ExtentCache( Extents(1:4c9e444c:::smithi13140796-48:head)[ Extent(0~425984:7) ) 2019-12-07T19:33:58.778+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] try_reads_to_commit: written: {1:4c9e444c:::smithi13140796-48:head={0~425984(425984)}} 2019-12-07T19:33:58.778+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] try_reads_to_commit: op: Op(1:4c9e444c:::smithi13140796-48:head v=21'7 tt=0'0 tid=8 reqid=client.4540.0:142 client_op=osd_op(client.4540.0:142 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~425984 in=425984b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'5 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[0~425984]}) 2019-12-07T19:33:58.778+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] try_reads_to_commit: written_set: {1:4c9e444c:::smithi13140796-48:head=[0~425984]} 2019-12-07T19:33:58.778+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] try_reads_to_commit: 1:4c9e444c:::smithi13140796-48:head,{0~425984(425984)} 2019-12-07T19:33:58.778+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 21/18 ECSubWrite(tid=8, reqid=client.4540.0:142, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 -- 0x558e83fea800 con 0x558e818d7c00 2019-12-07T19:33:58.778+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 21/18 ECSubWrite(tid=8, reqid=client.4540.0:142, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 -- 0x558e83fed000 con 0x558e818d7000 2019-12-07T19:33:58.778+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 21/18 ECSubWrite(tid=8, reqid=client.4540.0:142, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 -- 0x558e83fea000 con 0x558e818fe400 2019-12-07T19:33:58.778+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s4 21/18 ECSubWrite(tid=8, reqid=client.4540.0:142, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 -- 0x558e83fec000 con 0x558e818d6800 2019-12-07T19:33:58.778+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=8, reqid=client.4540.0:142, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 -- 0x558e83fec800 con 0x558e818ffc00 2019-12-07T19:33:58.778+0000 7fe2caa6c700 20 earliest_dup_version = 0 2019-12-07T19:33:58.778+0000 7fe2caa6c700 20 trim 21'7 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:142 2019-12-07T19:33:58.777249+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.778+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] append_log log((0'0,21'6], crt=21'3) [21'7 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:142 2019-12-07T19:33:58.777249+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.778+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 lua=21'6 crt=21'3 lcod 21'4 mlcod 21'4 active+clean] add_log_entry 21'7 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:142 2019-12-07T19:33:58.777249+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.778+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 lua=21'6 crt=21'5 lcod 21'4 mlcod 21'4 active+clean] rollforward: entry=21'4 (21'2) modify 1:4986b721:::smithi13140796-43:head by client.4540.0:128 2019-12-07T19:33:58.750475+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.778+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 lua=21'6 crt=21'5 lcod 21'4 mlcod 21'4 active+clean] rollforward: entry=21'5 (21'1) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:116 2019-12-07T19:33:58.731948+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 lua=21'6 crt=21'5 lcod 21'4 mlcod 21'4 active+clean] append_log approx pg log length = 7 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 lua=21'6 crt=21'5 lcod 21'4 mlcod 21'4 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.779+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e8195e900 seq 12 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:4c9e444c:::smithi13140796-48:head# key 0x8080000000000000014c9e44'L!smithi13140796-48!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.779+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1104 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# = 0 2019-12-07T19:33:58.779+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 2 keys 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 2 keys = 0 2019-12-07T19:33:58.779+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.779+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 0x0~1a000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 0#1:4c9e444c:::smithi13140796-48:head# 0x0~1a000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1a000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8195e900 13 blobs 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 fbmap_alloc 0x558e80233300 allocate 0x1a000/1000,1a000,0 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x273000~1a000/1000,1a000,0 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x273000~1a000] 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83edee70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edefc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83edee70 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edefc0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83edee70) get_ref 0x0~2000 Blob(0x558e83edee70 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edefc0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83edee70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83edee70 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edefc0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83edefd8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd2f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edf730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd2f50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf730 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd2f50) get_ref 0x0~2000 Blob(0x558e83dd2f50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf730 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd2f50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83dd2f50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edf730 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83edf748 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6fb90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fa40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6fb90 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fa40 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fb90) get_ref 0x0~2000 Blob(0x558e83e6fb90 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fa40 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fb90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83e6fb90 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6fa40 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fa58 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6fc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6fc00 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fab0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fc00) get_ref 0x0~2000 Blob(0x558e83e6fc00 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fab0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fc00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83e6fc00 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6fab0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fac8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6fc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6fc70 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fb20 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fc70) get_ref 0x0~2000 Blob(0x558e83e6fc70 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fb20 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fc70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83e6fc70 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6fb20 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fb38 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6fe30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6fe30 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fce0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fe30) get_ref 0x0~2000 Blob(0x558e83e6fe30 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fce0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fe30) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83e6fe30 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6fce0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fcf8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6fea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6fea0 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fd50 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fea0) get_ref 0x0~2000 Blob(0x558e83e6fea0 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fd50 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fea0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83e6fea0 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6fd50 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fd68 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6ff10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fdc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6ff10 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fdc0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6ff10) get_ref 0x0~2000 Blob(0x558e83e6ff10 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fdc0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6ff10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83e6ff10 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6fdc0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fdd8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6ff80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f48000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6ff80 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48000 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6ff80) get_ref 0x0~2000 Blob(0x558e83e6ff80 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48000 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6ff80) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83e6ff80 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48000 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48018 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f48070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f480e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f48070 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f480e0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48070) get_ref 0x0~2000 Blob(0x558e83f48070 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f480e0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48070) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83f48070 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f480e0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f480f8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f48150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f481c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f48150 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f481c0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48150) get_ref 0x0~2000 Blob(0x558e83f48150 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f481c0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48150) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83f48150 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f481c0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f481d8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f48230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f482a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f48230 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f482a0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48230) get_ref 0x0~2000 Blob(0x558e83f48230 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f482a0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48230) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83f48230 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f482a0 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f482b8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f48310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f48380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f48310 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48380 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48310) get_ref 0x0~2000 Blob(0x558e83f48310 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48380 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48310) get_ref init 0x2000, 1000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83f48310 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48380 sbid 0x0)) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48398 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1a000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e81a03a08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 0x0~1a000 = 0 2019-12-07T19:33:58.779+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.779+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e900 cost 216118 (27 ios * 4000 + 108118 bytes) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e900 onodes 0x558e81a03900 shared_blobs 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e81a03a08) update 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e81a03a08) update inline shard 250 bytes from 13 extents 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:4c9e444c:::smithi13140796-48:head# is 669 (413 bytes onode + 2 bytes spanning blobs + 254 bytes inline extents) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e900 allocated 0x[273000~1a000] released 0x[] 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 freelist allocate 0x273000~1a000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 freelist _xor first_key 0x200000 last_key 0x280000 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 prepare 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8195e900 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 lua=21'6 crt=21'5 lcod 21'4 mlcod 21'4 active+clean] op_applied version 21'7 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'5 lcod 21'4 mlcod 21'4 active+clean] eval_repop repgather(0x558e81123e60 21'7 rep_tid=8 committed?=0 r=0) 2019-12-07T19:33:58.779+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e84014000 finish 2019-12-07T19:33:58.779+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e840142c0 prio 196 cost 0 latency 0.001086 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'5 lcod 21'4 mlcod 21'4 active+clean] 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'5 lcod 21'4 mlcod 21'4 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=5, last_complete=21'6, committed=1, applied=1)) v2 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'5 lcod 21'4 mlcod 21'4 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:41fe8a11:::smithi13140796-45:head v=21'6 tt=0'0 tid=5 reqid=client.4540.0:134 client_op=osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'3 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41fe8a11:::smithi13140796-45:head=[]}) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'5 crt=21'5 lcod 21'4 mlcod 21'4 active+clean] repop_all_committed: repop tid 5 all committed 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] eval_repop repgather(0x558e81123680 21'6 rep_tid=5 committed?=1 r=0) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] commit: repgather(0x558e81123680 21'6 rep_tid=5 committed?=1 r=0) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] log_op_stats osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 0 outb 0 lat 0.017704 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] sending reply on osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x558e83b08000 2019-12-07T19:33:58.779+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 aio_wait 2019-12-07T19:33:58.779+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e900 2019-12-07T19:33:58.779+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 io_done 2019-12-07T19:33:58.779+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(134 smithi13140796-45 [setxattr (56)] v21'6 uv6 ondisk = 0) v8 -- 0x558e83b08000 con 0x558e83d68000 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.779+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] publish_stats_to_osd 21:20 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] removing repgather(0x558e81123680 21'6 rep_tid=5 committed?=1 r=0) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] q front is repgather(0x558e81123680 21'6 rep_tid=5 committed?=1 r=0) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] remove_repop repgather(0x558e81123680 21'6 rep_tid=5 committed?=1 r=0) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] requeue_op 0x558e81942dc0 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] try_finish_rmw: Op(1:41fe8a11:::smithi13140796-45:head v=21'6 tt=0'0 tid=5 reqid=client.4540.0:134 client_op=osd_op(client.4540.0:134 1.2s0 1:41fe8a11:::smithi13140796-45:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'3 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41fe8a11:::smithi13140796-45:head=[]}) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] try_finish_rmw: ExtentCache( Extents(1:4c9e444c:::smithi13140796-48:head)[ Extent(0~425984:7) ) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e840142c0 finish 2019-12-07T19:33:58.779+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.779+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.779+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81942dc0 prio 63 cost 0 latency 0.017843 osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] _handle_message: osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.779+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1410 < min_flush_size 524288 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op: op osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.779+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03900 had 1 2019-12-07T19:33:58.779+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:33:58.779+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1433 < min_flush_size 524288 2019-12-07T19:33:58.779+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1433 < min_flush_size 524288 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.779+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.779+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3cc3a~599 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.779+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3cc3a 0x3cc3a~599 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_object_context: found obc in cache: 0x558e83dd1200 2019-12-07T19:33:58.779+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.779+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xcc3a 2019-12-07T19:33:58.779+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc3a 2019-12-07T19:33:58.779+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.779+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x1d3 and padding block with 0xe2d 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_object_context: 0x558e83dd1200 1:41fe8a11:::smithi13140796-45:head rwstate(none n=0 w=0) oi: 1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e81123c20 snapset: 0=[]:{} 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] find_object_context 1:41fe8a11:::smithi13140796-45:head @head oi=1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op oi 1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op obc obc(1:41fe8a11:::smithi13140796-45:head rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] execute_ctx 0x558e83c48a00 2019-12-07T19:33:58.779+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3d1d3 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] execute_ctx 1:41fe8a11:::smithi13140796-45:head [read 0~1] ov 21'6 2019-12-07T19:33:58.779+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.779+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.779+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.779+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_osd_op 1:41fe8a11:::smithi13140796-45:head [read 0~1] 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_osd_op read 0~1 2019-12-07T19:33:58.779+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_read 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] async_read noted for 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.779+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000152605s (0.000042575s flush + 0.000110030s kv commit) 2019-12-07T19:33:58.779+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_all_avail_shards: checking acting 0(0) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_all_avail_shards: checking acting 4(1) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_all_avail_shards: checking acting 5(4) 2019-12-07T19:33:58.779+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 11 ==== osd_op(client.4540.0:143 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 226+0+376832 (crc 0 0 0) 0x558e83da6b00 con 0x558e83d68000 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_all_avail_shards: checking acting 6(3) 2019-12-07T19:33:58.779+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0af20 prio 63 cost 376832 latency 0.000018 epoch 21 osd_op(client.4540.0:143 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.779+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:143 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 376832 e21) 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] start_read_op: starting ReadOp(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.779+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 12 ==== osd_op(client.4540.0:144 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8 ==== 226+0+63 (crc 0 0 0) 0x558e83da62c0 con 0x558e83d68000 2019-12-07T19:33:58.779+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_read_op: starting read ReadOp(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.779+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0b600 prio 63 cost 63 latency 0.000008 epoch 21 osd_op(client.4540.0:144 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.779+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:144 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) 2019-12-07T19:33:58.779+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 13 ==== osd_op(client.4540.0:145 1.2s0 1.32227932 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 ==== 226+0+0 (crc 0 0 0) 0x558e83da6000 con 0x558e83d68000 2019-12-07T19:33:58.780+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e83b09680 con 0x558e802a5400 2019-12-07T19:33:58.780+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0b4a0 prio 63 cost 0 latency 0.000016 epoch 21 osd_op(client.4540.0:145 1.2s0 1.32227932 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.780+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:145 1.2s0 1.32227932 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.780+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.2s2 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402ec80 con 0x558e818d7c00 2019-12-07T19:33:58.780+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.2s1 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402f900 con 0x558e818fe400 2019-12-07T19:33:58.780+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558e83b09680 con 0x558e802a5400 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e900> 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.780+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402fb80 con 0x558e818ffc00 2019-12-07T19:33:58.780+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 kv_submitted 2019-12-07T19:33:58.780+0000 7fe2d427f700 15 osd.0 21 enqueue_op 0x558e81942c60 prio 127 cost 0 latency 0.000035 epoch 21 MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e900 2019-12-07T19:33:58.780+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 finishing 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e900 onodes 0x558e81a03900 2019-12-07T19:33:58.780+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6fa58 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11e60 space 0x558e83e6fa58 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6fac8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11dd0 space 0x558e83e6fac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6fb38 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11d40 space 0x558e83e6fb38 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6fcf8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11cb0 space 0x558e83e6fcf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6fd68 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11c20 space 0x558e83e6fd68 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6fdd8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11b90 space 0x558e83e6fdd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edefd8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11050 space 0x558e83edefd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edf748 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11ef0 space 0x558e83edf748 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f48018 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10090 space 0x558e83f48018 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f480f8 in 0x558e802195e0) _finish_write discard buffer(0x558e802299e0 space 0x558e83f480f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f481d8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d113b0 space 0x558e83f481d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f482b8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11440 space 0x558e83f482b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f48398 in 0x558e802195e0) _finish_write discard buffer(0x558e83d115f0 space 0x558e83f48398 0x0~2000 writing nocache) 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_read_op: started ReadOp(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head,6(3)=1:41fe8a11:::smithi13140796-45:head}, in_progress=0(0),1(2),4(1),6(3)) 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e900 done 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:33:58.780+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e900 [] 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81942dc0 finish 2019-12-07T19:33:58.780+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.780+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81942c60 prio 127 cost 0 latency 0.000086 MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=0,4096,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.780+0000 7fe2c6a64700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.780+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 0x0~1000 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x2e000 (188416) 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840308c0 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84030930 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e840308c0 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84030930 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.780+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.780+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:143 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 376832 e21) queued 2019-12-07T19:33:58.780+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 14 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 ==== 4261+0+0 (unknown 0 0 0) 0x558e8402ec80 con 0x558e818ffc00 2019-12-07T19:33:58.780+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0b080 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.780+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.780+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 15 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83fee800 con 0x558e818ffc00 2019-12-07T19:33:58.780+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0ac60 prio 196 cost 0 latency 0.000008 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.780+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e840308c0 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84030930 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e84030948 in 0x558e802195e0) _discard 0x0~1000 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 _add level 1 near 0 on buffer(0x558e83d10750 space 0x558e84030948 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 0x0~1000 = 4096 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_hash_info: Getting attr on 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_hash_info: not in cache 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_hash_info: found on disk, size 188416 2019-12-07T19:33:58.780+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key = 0 2019-12-07T19:33:58.780+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 -- 0x558e819ce000 con 0x558e802a5400 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81942c60 finish 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.780+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558e819ce000 con 0x558e802a5400 2019-12-07T19:33:58.780+0000 7fe2d427f700 15 osd.0 21 enqueue_op 0x558e81942b00 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.780+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:143 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 376832 e21) pg 0x558e81075000 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0af20 prio 63 cost 376832 latency 0.000884 osd_op(client.4540.0:143 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] _handle_message: osd_op(client.4540.0:143 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op: op osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_object_context: found obc in cache: 0x558e83ff0000 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_object_context: 0x558e83ff0000 1:4c9e444c:::smithi13140796-48:head rwstate(excl n=1 w=0) oi: 1:4c9e444c:::smithi13140796-48:head(21'7 client.4540.0:142 dirty|data_digest s 425984 uv 7 dd b014a64c alloc_hint [0 0 0]) exists: 1 ssc: 0x558e81123d40 snapset: 0=[]:{} 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] find_object_context 1:4c9e444c:::smithi13140796-48:head @head oi=1:4c9e444c:::smithi13140796-48:head(21'7 client.4540.0:142 dirty|data_digest s 425984 uv 7 dd b014a64c alloc_hint [0 0 0]) 2019-12-07T19:33:58.780+0000 7fe2c6a64700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op oi 1:4c9e444c:::smithi13140796-48:head(21'7 client.4540.0:142 dirty|data_digest s 425984 uv 7 dd b014a64c alloc_hint [0 0 0]) 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0af20 finish 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0ac60 prio 196 cost 0 latency 0.000280 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0ac60 finish 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.780+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b080 prio 127 cost 0 latency 0.000343 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.780+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=9, attrs_read=0) 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [6(3),4096]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head,6(3)=1:41fe8a11:::smithi13140796-45:head}, in_progress=0(0),1(2),4(1)) 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b080 finish 2019-12-07T19:33:58.781+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.781+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:144 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) queued 2019-12-07T19:33:58.781+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.781+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81942b00 prio 127 cost 0 latency 0.000257 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=9, attrs_read=0) 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [0(0),4096, 6(3),4096]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head,6(3)=1:41fe8a11:::smithi13140796-45:head}, in_progress=1(2),4(1)) 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81942b00 finish 2019-12-07T19:33:58.781+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.781+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:145 1.2s0 1.32227932 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.781+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.781+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:144 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) pg 0x558e81075000 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b600 prio 63 cost 63 latency 0.001137 osd_op(client.4540.0:144 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] _handle_message: osd_op(client.4540.0:144 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.781+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op: op osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.781+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_object_context: found obc in cache: 0x558e83ff0000 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_object_context: 0x558e83ff0000 1:4c9e444c:::smithi13140796-48:head rwstate(excl n=1 w=1) oi: 1:4c9e444c:::smithi13140796-48:head(21'7 client.4540.0:142 dirty|data_digest s 425984 uv 7 dd b014a64c alloc_hint [0 0 0]) exists: 1 ssc: 0x558e81123d40 snapset: 0=[]:{} 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] find_object_context 1:4c9e444c:::smithi13140796-48:head @head oi=1:4c9e444c:::smithi13140796-48:head(21'7 client.4540.0:142 dirty|data_digest s 425984 uv 7 dd b014a64c alloc_hint [0 0 0]) 2019-12-07T19:33:58.781+0000 7fe2c6a64700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op oi 1:4c9e444c:::smithi13140796-48:head(21'7 client.4540.0:142 dirty|data_digest s 425984 uv 7 dd b014a64c alloc_hint [0 0 0]) 2019-12-07T19:33:58.781+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.781+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b600 finish 2019-12-07T19:33:58.781+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.781+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.781+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:145 1.2s0 1.32227932 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.781+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b4a0 prio 63 cost 0 latency 0.001321 osd_op(client.4540.0:145 1.2s0 1.32227932 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.781+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] _handle_message: osd_op(client.4540.0:145 1.2s0 1.32227932 (undecoded) ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.781+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op: op osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.781+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.781+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.781+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_object_context: found obc in cache: 0x558e83ff0000 2019-12-07T19:33:58.781+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] get_object_context: 0x558e83ff0000 1:4c9e444c:::smithi13140796-48:head rwstate(excl n=1 w=2) oi: 1:4c9e444c:::smithi13140796-48:head(21'7 client.4540.0:142 dirty|data_digest s 425984 uv 7 dd b014a64c alloc_hint [0 0 0]) exists: 1 ssc: 0x558e81123d40 snapset: 0=[]:{} 2019-12-07T19:33:58.781+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] find_object_context 1:4c9e444c:::smithi13140796-48:head @head oi=1:4c9e444c:::smithi13140796-48:head(21'7 client.4540.0:142 dirty|data_digest s 425984 uv 7 dd b014a64c alloc_hint [0 0 0]) 2019-12-07T19:33:58.781+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op oi 1:4c9e444c:::smithi13140796-48:head(21'7 client.4540.0:142 dirty|data_digest s 425984 uv 7 dd b014a64c alloc_hint [0 0 0]) 2019-12-07T19:33:58.781+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.781+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b4a0 finish 2019-12-07T19:33:58.781+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.782+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 12 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818a9000 con 0x558e818d6800 2019-12-07T19:33:58.782+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0adc0 prio 196 cost 0 latency 0.000026 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.782+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.782+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 17 ==== MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=25, reqid=client.4540.0:136, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83fec000 con 0x558e818d6000 2019-12-07T19:33:58.782+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e81943760 prio 127 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=25, reqid=client.4540.0:136, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.782+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=25, reqid=client.4540.0:136, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.782+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 18 ==== MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:137, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e83cbd000 con 0x558e818d6000 2019-12-07T19:33:58.782+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e81943ce0 prio 127 cost 0 latency 0.000024 epoch 21 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:137, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.782+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:137, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=25, reqid=client.4540.0:136, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=25, reqid=client.4540.0:136, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558e819e7000 2019-12-07T19:33:58.782+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 40 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818a8600 con 0x558e818d7000 2019-12-07T19:33:58.782+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a2c0 prio 196 cost 0 latency 0.000019 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.782+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.782+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 29 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 ==== 4261+0+0 (unknown 0 0 0) 0x558e8402ec80 con 0x558e818fe400 2019-12-07T19:33:58.782+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0bce0 prio 127 cost 0 latency 0.000014 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.782+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.782+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 30 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e810fce00 con 0x558e818fe400 2019-12-07T19:33:58.782+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0be40 prio 196 cost 0 latency 0.000010 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.782+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 osd.0 21 dequeue_op 0x558e81943760 prio 127 cost 0 latency 0.000123 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=25, reqid=client.4540.0:136, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.ds5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=25, reqid=client.4540.0:136, at_version=0'0, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'5], crt=21'4) [] 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 osd.0 pg_epoch: 21 pg[1.ds5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] rollforward: entry=21'5 (0'0) modify 1:be42858c:::smithi13140796-46:head by client.4540.0:136 2019-12-07T19:33:58.768176+0000 0 ObjectCleanRegions clean_offsets: [770048~18446744073708781567], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.782+0000 7fe2c5261700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e811fac00 seq 13 2019-12-07T19:33:58.782+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fac00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fac00 onodes shared_blobs 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fac00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 prepare 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fac00 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 io_done 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 osd.0 21 dequeue_op 0x558e81943760 finish 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:137, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.782+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:137, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558e819e7000 2019-12-07T19:33:58.782+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.782+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 osd.0 21 dequeue_op 0x558e81943ce0 prio 127 cost 0 latency 0.000337 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:137, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.ds5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.782+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=26, reqid=client.4540.0:137, at_version=21'6, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.782+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'5], crt=21'5) [21'6 (21'5) modify 1:be42858c:::smithi13140796-46:head by client.4540.0:137 2019-12-07T19:33:58.768215+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3d1d3~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3d1d3 0x3d1d3~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xd1d3 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x1d3 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.783+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] add_log_entry 21'6 (21'5) modify 1:be42858c:::smithi13140796-46:head by client.4540.0:137 2019-12-07T19:33:58.768215+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x644 and padding block with 0x9bc 2019-12-07T19:33:58.783+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.783+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.783+0000 7fe2c5261700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.783+0000 7fe2c5261700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.783+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:58.783+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e81a0b800 seq 14 2019-12-07T19:33:58.783+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.ds5_head 5#1:be42858c:::smithi13140796-46:head# 3 keys 2019-12-07T19:33:58.783+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.ds5_head 5#1:be42858c:::smithi13140796-46:head# 3 keys = 0 2019-12-07T19:33:58.783+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.ds5_head 5#1:be42858c:::smithi13140796-46:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.783+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.ds5_head 5#1:be42858c:::smithi13140796-46:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.783+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:58.783+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.783+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.783+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.783+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.783+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:58.783+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b800 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.783+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b800 onodes 0x558e83cb3180 shared_blobs 2019-12-07T19:33:58.783+0000 7fe2c5261700 20 bluestore.extentmap(0x558e83cb3288) update 5#1:be42858c:::smithi13140796-46:head# 2019-12-07T19:33:58.783+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:be42858c:::smithi13140796-46:head# is 947 (485 bytes onode + 2 bytes spanning blobs + 460 bytes inline extents) 2019-12-07T19:33:58.783+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b800 allocated 0x[] released 0x[] 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3d644 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.783+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 prepare 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.783+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b800 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.783+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 io_done 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000137806s (0.000000148s flush + 0.000137658s kv commit) 2019-12-07T19:33:58.783+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 lua=21'5 crt=21'5 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.783+0000 7fe2c5261700 10 osd.0 21 dequeue_op 0x558e81943ce0 finish 2019-12-07T19:33:58.783+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1987 < min_flush_size 524288 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3180 had 1 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2010 < min_flush_size 524288 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2010 < min_flush_size 524288 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3d644~7da to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3d644 0x3d644~7da to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xd644 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x644 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xe1e and padding block with 0x1e2 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fac00> 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.783+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 kv_submitted 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fac00 2019-12-07T19:33:58.783+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 finishing 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fac00 onodes 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fac00 done 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 kv_submitted 2019-12-07T19:33:58.783+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fac00 [] 2019-12-07T19:33:58.783+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.783+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 75 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 ==== 4261+0+0 (unknown 0 0 0) 0x558e8402fb80 con 0x558e818d7c00 2019-12-07T19:33:58.783+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=25, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558e818ef400 con 0x558e818d6000 2019-12-07T19:33:58.783+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0b8c0 prio 127 cost 0 latency 0.000015 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.783+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.783+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.783+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3de1e 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.783+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000118466s (0.000000081s flush + 0.000118385s kv commit) 2019-12-07T19:33:58.783+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0b800> 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.783+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 kv_submitted 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b800 2019-12-07T19:33:58.783+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 finishing 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b800 onodes 0x558e83cb3180 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 done 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:33:58.783+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b800 [] 2019-12-07T19:33:58.783+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.783+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.783+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=26, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558e819dec00 con 0x558e818d6000 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.783+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.783+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0adc0 prio 196 cost 0 latency 0.000738 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0adc0 finish 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0a2c0 prio 196 cost 0 latency 0.000599 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0a2c0 finish 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0be40 prio 196 cost 0 latency 0.000566 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0be40 finish 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0bce0 prio 127 cost 0 latency 0.000615 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=9, attrs_read=0) 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [0(0),4096, 4(1),4096, 6(3),4096]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head,6(3)=1:41fe8a11:::smithi13140796-45:head}, in_progress=1(2)) 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0bce0 finish 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b8c0 prio 127 cost 0 latency 0.000247 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=9, attrs_read=0)) v2 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=9, attrs_read=0) 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply have shard=0 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply have shard=2 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply have shard=1 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply have shard=3 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_read_reply Complete: ReadOp(tid=9, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [0(0),4096, 1(2),4096, 4(1),4096, 6(3),4096]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head,6(3)=1:41fe8a11:::smithi13140796-45:head}, in_progress=) 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] objects_read_async_cb: got: {1:41fe8a11:::smithi13140796-45:head=0,{0~16384(16384)}} 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] objects_read_async_cb: cache: ExtentCache( Extents(1:4c9e444c:::smithi13140796-48:head)[ Extent(0~425984:7) ) 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] execute_ctx 0x558e83c48a00 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] execute_ctx 1:41fe8a11:::smithi13140796-45:head [read 0~1 out=1b] ov 21'6 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_osd_op 1:41fe8a11:::smithi13140796-45:head [read 0~1 out=1b] 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_osd_op read 0~1 out=1b 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] execute_ctx alloc reply 0x558e8402ec80 result 0 2019-12-07T19:33:58.783+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:33:58.783+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] log_op_stats osd_op(client.4540.0:135 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~1 out=1b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 inb 0 outb 1 lat 0.021637 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.783+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] publish_stats_to_osd 21:21 2019-12-07T19:33:58.783+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(135 smithi13140796-45 [read 0~1 out=1b] v0'0 uv6 ondisk = 0) v8 -- 0x558e8402ec80 con 0x558e83d68000 2019-12-07T19:33:58.783+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b8c0 finish 2019-12-07T19:33:58.783+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.785+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 76 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84029000 con 0x558e818d7c00 2019-12-07T19:33:58.785+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0ba20 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.785+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0ba20 prio 196 cost 0 latency 0.000055 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=8, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:4c9e444c:::smithi13140796-48:head v=21'7 tt=0'0 tid=8 reqid=client.4540.0:142 client_op=osd_op(client.4540.0:142 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~425984 in=425984b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'5 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[0~425984]}) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'6 crt=21'5 lcod 21'5 mlcod 21'5 active+clean] repop_all_committed: repop tid 8 all committed 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] eval_repop repgather(0x558e81123e60 21'7 rep_tid=8 committed?=1 r=0) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] commit: repgather(0x558e81123e60 21'7 rep_tid=8 committed?=1 r=0) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] log_op_stats osd_op(client.4540.0:142 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~425984 in=425984b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 425984 outb 0 lat 0.008701 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] sending reply on osd_op(client.4540.0:142 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~425984 in=425984b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x558e83cb3b80 2019-12-07T19:33:58.785+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(142 smithi13140796-48 [append 0~425984,stat] v21'7 uv7 ondisk = 0) v8 -- 0x558e83cb3b80 con 0x558e83d68000 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.785+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] publish_stats_to_osd 21:22 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] removing repgather(0x558e81123e60 21'7 rep_tid=8 committed?=1 r=0) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] q front is repgather(0x558e81123e60 21'7 rep_tid=8 committed?=1 r=0) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] remove_repop repgather(0x558e81123e60 21'7 rep_tid=8 committed?=1 r=0) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] requeue_op 0x558e83b0b4a0 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] requeue_op 0x558e83b0b600 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] requeue_op 0x558e83b0af20 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 376832 e21) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] try_finish_rmw: Op(1:4c9e444c:::smithi13140796-48:head v=21'7 tt=0'0 tid=8 reqid=client.4540.0:142 client_op=osd_op(client.4540.0:142 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~425984 in=425984b,stat out=16b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'5 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[0~425984]}) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] try_finish_rmw: ExtentCache( Extents(1:4c9e444c:::smithi13140796-48:head)[ Extent(0~425984:7) ) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: starting commit on Op(1:4c9e444c:::smithi13140796-48:head v=0'0 tt=0'0 tid=10 reqid=client.4540.0:142 roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: written: {} 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: op: Op(1:4c9e444c:::smithi13140796-48:head v=0'0 tt=0'0 tid=10 reqid=client.4540.0:142 roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: written_set: {} 2019-12-07T19:33:58.785+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 21/18 ECSubWrite(tid=10, reqid=client.4540.0:142, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e83ac9000 con 0x558e818d7c00 2019-12-07T19:33:58.785+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 21/18 ECSubWrite(tid=10, reqid=client.4540.0:142, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e83cbe000 con 0x558e818d7000 2019-12-07T19:33:58.785+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 21/18 ECSubWrite(tid=10, reqid=client.4540.0:142, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e83cbf800 con 0x558e818fe400 2019-12-07T19:33:58.785+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s4 21/18 ECSubWrite(tid=10, reqid=client.4540.0:142, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e83feb000 con 0x558e818d6800 2019-12-07T19:33:58.785+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=10, reqid=client.4540.0:142, at_version=0'0, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e819ce000 con 0x558e818ffc00 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'5 lcod 21'6 mlcod 21'6 active+clean] append_log log((0'0,21'7], crt=21'5) [] 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] rollforward: entry=21'6 (21'3) modify 1:41fe8a11:::smithi13140796-45:head by client.4540.0:134 2019-12-07T19:33:58.762144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] rollforward: entry=21'7 (0'0) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:142 2019-12-07T19:33:58.777249+0000 0 ObjectCleanRegions clean_offsets: [425984~18446744073709125631], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] append_log approx pg log length = 7 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.785+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e8195fb00 seq 13 2019-12-07T19:33:58.785+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195fb00 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195fb00 onodes shared_blobs 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195fb00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 prepare 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195fb00 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 io_done 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0ba20 finish 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 376832 e21) queued 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 376832 e21) pg 0x558e81075000 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0af20 prio 63 cost 376832 latency 0.005975 osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 pg pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op: op osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.785+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_object_context: found obc in cache: 0x558e83ff0000 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_object_context: 0x558e83ff0000 1:4c9e444c:::smithi13140796-48:head rwstate(none n=0 w=0) oi: 1:4c9e444c:::smithi13140796-48:head(21'7 client.4540.0:142 dirty|data_digest s 425984 uv 7 dd b014a64c alloc_hint [0 0 0]) exists: 1 ssc: 0x558e81123d40 snapset: 0=[]:{} 2019-12-07T19:33:58.785+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] find_object_context 1:4c9e444c:::smithi13140796-48:head @head oi=1:4c9e444c:::smithi13140796-48:head(21'7 client.4540.0:142 dirty|data_digest s 425984 uv 7 dd b014a64c alloc_hint [0 0 0]) 2019-12-07T19:33:58.786+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op oi 1:4c9e444c:::smithi13140796-48:head(21'7 client.4540.0:142 dirty|data_digest s 425984 uv 7 dd b014a64c alloc_hint [0 0 0]) 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op obc obc(1:4c9e444c:::smithi13140796-48:head rwstate(write n=1 w=0)) 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] execute_ctx 0x558e83c46000 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] execute_ctx 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] ov 21'7 av 21'8 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_osd_op 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_osd_op append 0~376832 in=376832b 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_osd_op 1:4c9e444c:::smithi13140796-48:head [write 425984~376832 in=376832b] 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_osd_op write 425984~376832 in=376832b 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] clean_regions modifiedclean_offsets: [0~425984,802816~18446744073708748799], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] make_writeable 1:4c9e444c:::smithi13140796-48:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] make_writeable 1:4c9e444c:::smithi13140796-48:head done, snapset=0=[]:{} 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] finish_ctx 1:4c9e444c:::smithi13140796-48:head 0x558e83c46000 op modify 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] set mtime to 2019-12-07T19:33:58.780115+0000 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] final snapset 0=[]:{} in 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] finish_ctx object 1:4c9e444c:::smithi13140796-48:head marks clean_regions clean_offsets: [0~425984,802816~18446744073708748799], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] execute_ctx alloc reply 0x558e8402ef00 result 0 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] calc_trim_to_aggressive limit = 21'7 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] op order client.4540 tid 143 last was 142 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] new_repop rep_tid 11 on osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] new_repop: repgather(0x558e81123e60 0'0 rep_tid=11 committed?=0 r=0) 2019-12-07T19:33:58.786+0000 7fe2caa6c700 7 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] issue_repop rep_tid 11 o 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] submit_transaction: op Op(1:4c9e444c:::smithi13140796-48:head v=21'8 tt=0'0 tid=11 reqid=client.4540.0:143 client_op=osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_hash_info: Getting attr on 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_hash_info: not in cache 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.786+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 31 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84029200 con 0x558e818fe400 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:33:58.786+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0b1e0 prio 196 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.786+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_hash_info: found on disk, size 106496 2019-12-07T19:33:58.786+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# hinfo_key 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# hinfo_key = 0 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): 1:4c9e444c:::smithi13140796-48:head projected size 802816 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] start_rmw: Op(1:4c9e444c:::smithi13140796-48:head v=21'8 tt=0'0 tid=11 reqid=client.4540.0:143 client_op=osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[425984~376832]}) 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_state_to_reads: Op(1:4c9e444c:::smithi13140796-48:head v=21'8 tt=0'0 tid=11 reqid=client.4540.0:143 client_op=osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[425984~376832]}) 2019-12-07T19:33:58.786+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: starting commit on Op(1:4c9e444c:::smithi13140796-48:head v=21'8 tt=0'0 tid=11 reqid=client.4540.0:143 client_op=osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[425984~376832]}) 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: ExtentCache( Extents(1:4c9e444c:::smithi13140796-48:head)[ Extent(425984~376832:8) ) 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] generate_transactions: 1:4c9e444c:::smithi13140796-48:head, current size is 425984 buffers are {425984~376832(376832)} 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): new_size start 425984 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): adding buffer_update 425984,376832 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): to_append: {425984~376832(376832)} 2019-12-07T19:33:58.786+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): appending 425984~376832 2019-12-07T19:33:58.786+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.786+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.786+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.786+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:33:58.786+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.786+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.786+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.786+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3de1e~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.786+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3de1e 0x3de1e~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.786+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.786+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xde1e 2019-12-07T19:33:58.786+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xe1e 2019-12-07T19:33:58.786+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.786+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x28f and padding block with 0xd71 2019-12-07T19:33:58.786+0000 7fe2e6d3a700 0 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 msgr2=0x558e81847180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2019-12-07T19:33:58.786+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 41 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028800 con 0x558e818d7000 2019-12-07T19:33:58.786+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943a20 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.786+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.786+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 msgr2=0x558e81847180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 95 2019-12-07T19:33:58.786+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 msgr2=0x558e81847180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:33:58.786+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=READY pgs=20 cs=2 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:33:58.786+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3e28f 2019-12-07T19:33:58.786+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.786+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.786+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.786+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.786+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000305697s (0.000000112s flush + 0.000305585s kv commit) 2019-12-07T19:33:58.786+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.786+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=READY pgs=20 cs=2 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:33:58.786+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.786+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195fb00> 2019-12-07T19:33:58.786+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.786+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 kv_submitted 2019-12-07T19:33:58.786+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195fb00 2019-12-07T19:33:58.786+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 finishing 2019-12-07T19:33:58.786+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195fb00 onodes 2019-12-07T19:33:58.786+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 done 2019-12-07T19:33:58.786+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:33:58.786+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195fb00 [] 2019-12-07T19:33:58.786+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.786+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] encode_and_write: 1:4c9e444c:::smithi13140796-48:head new_size 802816 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): 1:4c9e444c:::smithi13140796-48:head resetting hinfo to logical size 802816 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): marking append 425984 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: ExtentCache( Extents(1:4c9e444c:::smithi13140796-48:head)[ Extent(425984~376832:8) ) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: written: {1:4c9e444c:::smithi13140796-48:head={425984~376832(376832)}} 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: op: Op(1:4c9e444c:::smithi13140796-48:head v=21'8 tt=0'0 tid=11 reqid=client.4540.0:143 client_op=osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[425984~376832]}) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: written_set: {1:4c9e444c:::smithi13140796-48:head=[425984~376832]} 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: 1:4c9e444c:::smithi13140796-48:head,{425984~376832(376832)} 2019-12-07T19:33:58.787+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 21/18 ECSubWrite(tid=11, reqid=client.4540.0:143, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e819ce800 con 0x558e818d7c00 2019-12-07T19:33:58.787+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 21/18 ECSubWrite(tid=11, reqid=client.4540.0:143, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e819cf000 con 0x558e818d7000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 21/18 ECSubWrite(tid=11, reqid=client.4540.0:143, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e819cf800 con 0x558e818fe400 2019-12-07T19:33:58.787+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s4 21/18 ECSubWrite(tid=11, reqid=client.4540.0:143, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e819d0000 con 0x558e818d6800 2019-12-07T19:33:58.787+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=STANDBY pgs=20 cs=3 l=0 rx=0 tx=0).connect 2019-12-07T19:33:58.787+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=11, reqid=client.4540.0:143, at_version=21'8, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e819d0800 con 0x558e818ffc00 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 earliest_dup_version = 0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 trim 21'8 (21'7) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:143 2019-12-07T19:33:58.780115+0000 0 ObjectCleanRegions clean_offsets: [0~425984,802816~18446744073708748799], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] append_log log((0'0,21'7], crt=21'7) [21'8 (21'7) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:143 2019-12-07T19:33:58.780115+0000 0 ObjectCleanRegions clean_offsets: [0~425984,802816~18446744073708748799], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 lua=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] add_log_entry 21'8 (21'7) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:143 2019-12-07T19:33:58.780115+0000 0 ObjectCleanRegions clean_offsets: [0~425984,802816~18446744073708748799], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 lua=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] append_log approx pg log length = 8 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 lua=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e8195e300 seq 14 2019-12-07T19:33:58.787+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 2 keys 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 2 keys = 0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 0x1a000~17000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 0#1:4c9e444c:::smithi13140796-48:head# 0x1a000~17000 - have 0x1a000 (106496) bytes fadvise_flags 0x0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x1a000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8195e300 12 blobs 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 fbmap_alloc 0x558e80233300 allocate 0x17000/1000,17000,0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x28d000~17000/1000,17000,0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x28d000~17000] 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f489a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edfa40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f489a0 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edfa40 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f489a0) get_ref 0x0~2000 Blob(0x558e83f489a0 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edfa40 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f489a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83f489a0 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edfa40 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83edfa58 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f48a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f48a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f48a10 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48a80 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48a10) get_ref 0x0~2000 Blob(0x558e83f48a10 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48a80 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48a10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e83f48a10 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48a80 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48a98 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f49490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f492d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f49490 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f492d0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49490) get_ref 0x0~2000 Blob(0x558e83f49490 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f492d0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49490) get_ref init 0x2000, 1000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e83f49490 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f492d0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f492e8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.787+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=BANNER_CONNECTING pgs=20 cs=3 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f49420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f49260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f49420 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f49260 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49420) get_ref 0x0~2000 Blob(0x558e83f49420 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f49260 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49420) get_ref init 0x2000, 1000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e83f49420 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f49260 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f49278 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f493b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f491f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f493b0 blob([0x295000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f491f0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f493b0) get_ref 0x0~2000 Blob(0x558e83f493b0 blob([0x295000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f491f0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f493b0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e83f493b0 blob([0x295000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f491f0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f49208 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f49340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f49030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f49340 blob([0x297000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f49030 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49340) get_ref 0x0~2000 Blob(0x558e83f49340 blob([0x297000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f49030 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e83f49340 blob([0x297000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f49030 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f49048 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f49180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f48fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f49180 blob([0x299000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48fc0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49180) get_ref 0x0~2000 Blob(0x558e83f49180 blob([0x299000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48fc0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e83f49180 blob([0x299000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48fc0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48fd8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f49110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f48f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f49110 blob([0x29b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48f50 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49110) get_ref 0x0~2000 Blob(0x558e83f49110 blob([0x29b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48f50 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e83f49110 blob([0x29b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48f50 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48f68 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f490a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f48ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f490a0 blob([0x29d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48ee0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f490a0) get_ref 0x0~2000 Blob(0x558e83f490a0 blob([0x29d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48ee0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f490a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558e83f490a0 blob([0x29d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48ee0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48ef8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f48e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f49500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f48e70 blob([0x29f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f49500 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48e70) get_ref 0x0~2000 Blob(0x558e83f48e70 blob([0x29f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f49500 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48e70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558e83f48e70 blob([0x29f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f49500 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f49518 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f49570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f495e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f49570 blob([0x2a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f495e0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49570) get_ref 0x0~2000 Blob(0x558e83f49570 blob([0x2a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f495e0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49570) get_ref init 0x2000, 1000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558e83f49570 blob([0x2a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f495e0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f495f8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f49650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f496c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f49650 blob([0x2a3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f496c0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49650) get_ref 0x0~1000 Blob(0x558e83f49650 blob([0x2a3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f496c0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49650) get_ref init 0x1000, 1000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x30000~1000: 0x0~1000 Blob(0x558e83f49650 blob([0x2a3000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f496c0 sbid 0x0)) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f496d8 in 0x558e802195e0) _discard 0x0~1000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x31000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e81a03a08) dirty_range mark inline shard dirty 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 0x1a000~17000 = 0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e300 cost 192186 (24 ios * 4000 + 96186 bytes) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e300 onodes 0x558e81a03900 shared_blobs 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e81a03a08) update 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e81a03a08) update inline shard 475 bytes from 25 extents 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:4c9e444c:::smithi13140796-48:head# is 894 (413 bytes onode + 2 bytes spanning blobs + 479 bytes inline extents) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e300 allocated 0x[28d000~17000] released 0x[] 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 freelist allocate 0x28d000~17000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 prepare 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8195e300 2019-12-07T19:33:58.787+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 crc :-1 s=START_CONNECT pgs=20 cs=4 l=0 rx=0 tx=0)._fault waiting 15.000000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 lua=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] op_applied version 21'8 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] eval_repop repgather(0x558e81123e60 21'8 rep_tid=11 committed?=0 r=0) 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0af20 finish 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b1e0 prio 196 cost 0 latency 0.001717 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b1e0 finish 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81943a20 prio 196 cost 0 latency 0.001204 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81943a20 finish 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) queued 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8) prio 63 cost 63 e21) pg 0x558e81075000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b600 prio 63 cost 63 latency 0.007971 osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 pg pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op: op osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.787+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 may_write -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_object_context: found obc in cache: 0x558e83ff0000 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_object_context: 0x558e83ff0000 1:4c9e444c:::smithi13140796-48:head rwstate(write n=1 w=0) oi: 1:4c9e444c:::smithi13140796-48:head(21'8 client.4540.0:143 dirty|data_digest s 802816 uv 8 dd 18c6bbbb alloc_hint [0 0 0]) exists: 1 ssc: 0x558e81123d40 snapset: 0=[]:{} 2019-12-07T19:33:58.787+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] find_object_context 1:4c9e444c:::smithi13140796-48:head @head oi=1:4c9e444c:::smithi13140796-48:head(21'8 client.4540.0:143 dirty|data_digest s 802816 uv 8 dd 18c6bbbb alloc_hint [0 0 0]) 2019-12-07T19:33:58.788+0000 7fe2caa6c700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op oi 1:4c9e444c:::smithi13140796-48:head(21'8 client.4540.0:143 dirty|data_digest s 802816 uv 8 dd 18c6bbbb alloc_hint [0 0 0]) 2019-12-07T19:33:58.788+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op obc obc(1:4c9e444c:::smithi13140796-48:head rwstate(write n=2 w=0)) 2019-12-07T19:33:58.788+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] execute_ctx 0x558e8122b100 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] execute_ctx 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] ov 21'8 av 21'9 snapc 0=[] snapset 0=[]:{} 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_osd_op 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_osd_op setxattr _header (56) in=63b 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] make_writeable 1:4c9e444c:::smithi13140796-48:head snapset=0=[]:{} snapc=0=[] 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] make_writeable 1:4c9e444c:::smithi13140796-48:head done, snapset=0=[]:{} 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] finish_ctx 1:4c9e444c:::smithi13140796-48:head 0x558e8122b100 op modify 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] set mtime to 2019-12-07T19:33:58.780144+0000 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] final snapset 0=[]:{} in 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] finish_ctx object 1:4c9e444c:::smithi13140796-48:head marks clean_regions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] execute_ctx alloc reply 0x558e83c72f00 result 0 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] calc_trim_to_aggressive limit = 21'7 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] op order client.4540 tid 144 last was 143 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] new_repop rep_tid 12 on osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] new_repop: repgather(0x558e8023b7a0 0'0 rep_tid=12 committed?=0 r=0) 2019-12-07T19:33:58.788+0000 7fe2caa6c700 7 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] issue_repop rep_tid 12 o 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] submit_transaction: op Op(1:4c9e444c:::smithi13140796-48:head v=21'9 tt=0'0 tid=12 reqid=client.4540.0:144 client_op=osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_hash_info: Getting attr on 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): 1:4c9e444c:::smithi13140796-48:head projected size 802816 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] start_rmw: Op(1:4c9e444c:::smithi13140796-48:head v=21'9 tt=0'0 tid=12 reqid=client.4540.0:144 client_op=osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_state_to_reads: Op(1:4c9e444c:::smithi13140796-48:head v=21'9 tt=0'0 tid=12 reqid=client.4540.0:144 client_op=osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: starting commit on Op(1:4c9e444c:::smithi13140796-48:head v=21'9 tt=0'0 tid=12 reqid=client.4540.0:144 client_op=osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: ExtentCache( Extents(1:4c9e444c:::smithi13140796-48:head)[ Extent(425984~376832:8) ) 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] generate_transactions: 1:4c9e444c:::smithi13140796-48:head, current size is 802816 buffers are {} 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): new_size start 802816 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): to_overwrite: {} 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): to_append: {} 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] operator(): 1:4c9e444c:::smithi13140796-48:head resetting hinfo to logical size 802816 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: ExtentCache( Extents(1:4c9e444c:::smithi13140796-48:head)[ Extent(425984~376832:8) ) 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: written: {1:4c9e444c:::smithi13140796-48:head={}} 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: op: Op(1:4c9e444c:::smithi13140796-48:head v=21'9 tt=0'0 tid=12 reqid=client.4540.0:144 client_op=osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: written_set: {1:4c9e444c:::smithi13140796-48:head=[]} 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_reads_to_commit: 1:4c9e444c:::smithi13140796-48:head,{} 2019-12-07T19:33:58.788+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 21/18 ECSubWrite(tid=12, reqid=client.4540.0:144, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e819d1000 con 0x558e818d7c00 2019-12-07T19:33:58.788+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 21/18 ECSubWrite(tid=12, reqid=client.4540.0:144, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e819d1800 con 0x558e818d7000 2019-12-07T19:33:58.788+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 21/18 ECSubWrite(tid=12, reqid=client.4540.0:144, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e83e54000 con 0x558e818fe400 2019-12-07T19:33:58.788+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s4 21/18 ECSubWrite(tid=12, reqid=client.4540.0:144, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e83e54800 con 0x558e818d6800 2019-12-07T19:33:58.788+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=12, reqid=client.4540.0:144, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 -- 0x558e83e55000 con 0x558e818ffc00 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 earliest_dup_version = 0 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 trim 21'9 (21'8) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:144 2019-12-07T19:33:58.780144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] append_log log((0'0,21'8], crt=21'7) [21'9 (21'8) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:144 2019-12-07T19:33:58.780144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 lua=21'8 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] add_log_entry 21'9 (21'8) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:144 2019-12-07T19:33:58.780144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 lua=21'8 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] append_log approx pg log length = 9 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 lua=21'8 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.788+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.788+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 32 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028c00 con 0x558e818fe400 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e819a1200 seq 15 2019-12-07T19:33:58.788+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e819438c0 prio 196 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.788+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 3 keys 2019-12-07T19:33:58.788+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 3 keys = 0 2019-12-07T19:33:58.788+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.788+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes 0x558e81a03900 shared_blobs 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e81a03a08) update 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:4c9e444c:::smithi13140796-48:head# is 966 (485 bytes onode + 2 bytes spanning blobs + 479 bytes inline extents) 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[] released 0x[] 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 blocked by 0x558e8195e300 aio_wait 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 lua=21'8 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] op_applied version 21'9 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] eval_repop repgather(0x558e8023b7a0 21'9 rep_tid=12 committed?=0 r=0) 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0b600 finish 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.788+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.788+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b4a0 prio 63 cost 0 latency 0.008585 osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.788+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 aio_wait 2019-12-07T19:33:58.788+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e300 2019-12-07T19:33:58.788+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 io_done 2019-12-07T19:33:58.788+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:33:58.788+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e83d40c00 0x558e83ae2680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:33:58.788+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.788+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.788+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.788+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.788+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op: op osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.788+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.788+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.788+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_object_context: found obc in cache: 0x558e83ff0000 2019-12-07T19:33:58.788+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] get_object_context: 0x558e83ff0000 1:4c9e444c:::smithi13140796-48:head rwstate(write n=2 w=0) oi: 1:4c9e444c:::smithi13140796-48:head(21'9 client.4540.0:144 dirty|data_digest s 802816 uv 9 dd 18c6bbbb alloc_hint [0 0 0]) exists: 1 ssc: 0x558e81123d40 snapset: 0=[]:{} 2019-12-07T19:33:58.788+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2034 < min_flush_size 524288 2019-12-07T19:33:58.788+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] find_object_context 1:4c9e444c:::smithi13140796-48:head @head oi=1:4c9e444c:::smithi13140796-48:head(21'9 client.4540.0:144 dirty|data_digest s 802816 uv 9 dd 18c6bbbb alloc_hint [0 0 0]) 2019-12-07T19:33:58.788+0000 7fe2c6a64700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op oi 1:4c9e444c:::smithi13140796-48:head(21'9 client.4540.0:144 dirty|data_digest s 802816 uv 9 dd 18c6bbbb alloc_hint [0 0 0]) 2019-12-07T19:33:58.788+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03900 had 2 2019-12-07T19:33:58.788+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 2 2019-12-07T19:33:58.788+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] do_op waiting for rw locks 2019-12-07T19:33:58.788+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b4a0 finish 2019-12-07T19:33:58.788+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4040 < min_flush_size 524288 2019-12-07T19:33:58.788+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.788+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03900 had 1 2019-12-07T19:33:58.788+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:33:58.788+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4063 < min_flush_size 524288 2019-12-07T19:33:58.788+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4063 < min_flush_size 524288 2019-12-07T19:33:58.788+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.788+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3e28f~fdf to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.788+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3e28f 0x3e28f~fdf to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.788+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.788+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xe28f 2019-12-07T19:33:58.788+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x28f 2019-12-07T19:33:58.788+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.788+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x26e and padding block with 0xd92 2019-12-07T19:33:58.788+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> conn(0x558e83d40c00 0x558e83ae2680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e819438c0 prio 196 cost 0 latency 0.000329 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.788+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e819438c0 finish 2019-12-07T19:33:58.788+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.788+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 33 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028a00 con 0x558e818fe400 2019-12-07T19:33:58.788+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e81943a20 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.788+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.788+0000 7fe2e7d3c700 10 osd.0 21 ms_handle_authentication new session 0x558e818e9040 con 0x558e83d40c00 entity osd.3 addr 2019-12-07T19:33:58.788+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.788+0000 7fe2e7d3c700 10 osd.0 21 ms_handle_authentication session 0x558e818e9040 osd.3 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:33:58.788+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.788+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.788+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943a20 prio 196 cost 0 latency 0.000052 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.788+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.788+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943a20 finish 2019-12-07T19:33:58.788+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.789+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e83d40c00 0x558e83ae2680 crc :-1 s=SESSION_ACCEPTING pgs=26 cs=0 l=0 rx=0 tx=0).handle_reconnect reconnect to existing=0x558e818d7000 2019-12-07T19:33:58.789+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e83d40c00 0x558e83ae2680 crc :-1 s=SESSION_ACCEPTING pgs=26 cs=0 l=0 rx=0 tx=0).stop 2019-12-07T19:33:58.789+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] conn(0x558e818d7000 0x558e81847180 unknown :-1 s=READY pgs=26 cs=5 l=0 rx=0 tx=0).ready entity=osd.3 client_cookie=f8db806cc22eb7e9 server_cookie=47e95957322ca882 in_seq=41 out_seq=39 2019-12-07T19:33:58.789+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 16 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e811f7a00 con 0x558e818ffc00 2019-12-07T19:33:58.789+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0bb80 prio 196 cost 0 latency 0.000014 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.789+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.789+0000 7fe2d4a80700 2 osd.0 21 ms_handle_reset con 0x558e83d40c00 session 0x558e818e9040 2019-12-07T19:33:58.789+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3f26e 2019-12-07T19:33:58.789+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.789+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.789+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.789+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.789+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.001088227s (0.000045752s flush + 0.001042475s kv commit) 2019-12-07T19:33:58.789+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e300,0x558e819a1200> 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.789+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 kv_submitted 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e300 2019-12-07T19:33:58.789+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 finishing 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e300 onodes 0x558e81a03900 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edfa58 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11170 space 0x558e83edfa58 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f48a98 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11200 space 0x558e83f48a98 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f48ef8 in 0x558e802195e0) _finish_write discard buffer(0x558e83c8fc20 space 0x558e83f48ef8 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f48f68 in 0x558e802195e0) _finish_write discard buffer(0x558e83c8f290 space 0x558e83f48f68 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f48fd8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11320 space 0x558e83f48fd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f49048 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10360 space 0x558e83f49048 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f49208 in 0x558e802195e0) _finish_write discard buffer(0x558e83d10510 space 0x558e83f49208 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f49278 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11b00 space 0x558e83f49278 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f492e8 in 0x558e802195e0) _finish_write discard buffer(0x558e83d11290 space 0x558e83f492e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f49518 in 0x558e802195e0) _finish_write discard buffer(0x558e83c8e990 space 0x558e83f49518 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f495f8 in 0x558e802195e0) _finish_write discard buffer(0x558e83c8eab0 space 0x558e83f495f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f496d8 in 0x558e802195e0) _finish_write discard buffer(0x558e83c8ecf0 space 0x558e83f496d8 0x0~1000 writing nocache) 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 done 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 kv_submitted 2019-12-07T19:33:58.789+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e300 [] 2019-12-07T19:33:58.789+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.789+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:33:58.789+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 0x558e81a03900 2019-12-07T19:33:58.789+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0bb80 prio 196 cost 0 latency 0.000098 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:33:58.789+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:33:58.789+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.789+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.789+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.789+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0bb80 finish 2019-12-07T19:33:58.789+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.789+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 17 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e811f7200 con 0x558e818ffc00 2019-12-07T19:33:58.789+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83b0ab00 prio 196 cost 0 latency 0.000013 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.789+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.789+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.790+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0ab00 prio 196 cost 0 latency 0.000048 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.790+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.790+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0ab00 finish 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 18 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83feec00 con 0x558e818ffc00 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81942c60 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.790+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81942c60 prio 196 cost 0 latency 0.000090 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.790+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.790+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81942c60 finish 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 77 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84029800 con 0x558e818d7c00 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81942b00 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 78 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83fee000 con 0x558e818d7c00 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81943600 prio 196 cost 0 latency 0.000008 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 79 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84029000 con 0x558e818d7c00 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81942420 prio 196 cost 0 latency 0.000007 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.790+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81942b00 prio 196 cost 0 latency 0.000042 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.790+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.790+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81942b00 finish 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.790+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81943600 prio 196 cost 0 latency 0.000052 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.790+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.790+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81943600 finish 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.790+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81942420 prio 196 cost 0 latency 0.000048 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.790+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.790+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81942420 finish 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 42 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028600 con 0x558e818d7000 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e81943e40 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 43 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83fee400 con 0x558e818d7000 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e819434a0 prio 196 cost 0 latency 0.000007 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.790+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.790+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943e40 prio 196 cost 0 latency 0.000036 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.790+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.790+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943e40 finish 2019-12-07T19:33:58.790+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.790+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e819434a0 prio 196 cost 0 latency 0.000056 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.790+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.790+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e819434a0 finish 2019-12-07T19:33:58.790+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.791+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 19 ==== MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=27, reqid=client.4540.0:137, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e80e53000 con 0x558e818d6000 2019-12-07T19:33:58.791+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e81942dc0 prio 127 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=27, reqid=client.4540.0:137, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.791+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=27, reqid=client.4540.0:137, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.791+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.791+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=27, reqid=client.4540.0:137, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.791+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.791+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGOpItem(op=MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=27, reqid=client.4540.0:137, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x558e819e7000 2019-12-07T19:33:58.791+0000 7fe2c9269700 10 osd.0 21 dequeue_op 0x558e81942dc0 prio 127 cost 0 latency 0.000041 MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=27, reqid=client.4540.0:137, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.791+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.ds5 21/18 ECSubWrite(tid=27, reqid=client.4540.0:137, at_version=0'0, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.791+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'6], crt=21'5) [] 2019-12-07T19:33:58.791+0000 7fe2c9269700 20 osd.0 pg_epoch: 21 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] rollforward: entry=21'6 (21'5) modify 1:be42858c:::smithi13140796-46:head by client.4540.0:137 2019-12-07T19:33:58.768215+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.791+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.791+0000 7fe2c9269700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.791+0000 7fe2c9269700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.791+0000 7fe2c9269700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.791+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:33:58.791+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e83d4d500 seq 15 2019-12-07T19:33:58.791+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:33:58.791+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:33:58.791+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.791+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.791+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:33:58.791+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4d500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.791+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4d500 onodes shared_blobs 2019-12-07T19:33:58.791+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4d500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.791+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 prepare 2019-12-07T19:33:58.791+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4d500 2019-12-07T19:33:58.791+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 io_done 2019-12-07T19:33:58.791+0000 7fe2c9269700 10 osd.0 21 dequeue_op 0x558e81942dc0 finish 2019-12-07T19:33:58.791+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.791+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.791+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.791+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.791+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:33:58.791+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.791+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.791+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.791+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3f26e~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.791+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3f26e 0x3f26e~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.791+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.791+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xf26e 2019-12-07T19:33:58.791+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x26e 2019-12-07T19:33:58.791+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.791+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x6df and padding block with 0x921 2019-12-07T19:33:58.791+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3f6df 2019-12-07T19:33:58.791+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.791+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.791+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.791+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.791+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.791+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000086142s (0.000000081s flush + 0.000086061s kv commit) 2019-12-07T19:33:58.791+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4d500> 2019-12-07T19:33:58.791+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.791+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.791+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 kv_submitted 2019-12-07T19:33:58.791+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4d500 2019-12-07T19:33:58.791+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 finishing 2019-12-07T19:33:58.791+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4d500 onodes 2019-12-07T19:33:58.791+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4d500 done 2019-12-07T19:33:58.791+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:33:58.791+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4d500 [] 2019-12-07T19:33:58.791+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.791+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.791+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.791+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.ds0 21/18 ECSubWriteReply(tid=27, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558e83fef200 con 0x558e818d6000 2019-12-07T19:33:58.791+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:58.791+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 221925 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.792+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 13 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e811f7c00 con 0x558e818d6800 2019-12-07T19:33:58.792+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e81943a20 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.792+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943a20 prio 196 cost 0 latency 0.000101 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=10, last_complete=21'7, committed=1, applied=1)) v2 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_finish_rmw: Op(1:4c9e444c:::smithi13140796-48:head v=0'0 tt=0'0 tid=10 reqid=client.4540.0:142 roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] try_finish_rmw: ExtentCache( Extents(1:4c9e444c:::smithi13140796-48:head)[ Extent(425984~376832:8) ) 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943a20 finish 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.792+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 14 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818ef400 con 0x558e818d6800 2019-12-07T19:33:58.792+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e81942c60 prio 196 cost 0 latency 0.000011 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.792+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81942c60 prio 196 cost 0 latency 0.000031 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=11, last_complete=21'8, committed=1, applied=1)) v2 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:4c9e444c:::smithi13140796-48:head v=21'8 tt=0'0 tid=11 reqid=client.4540.0:143 client_op=osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[425984~376832]}) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'7 crt=21'7 lcod 21'6 mlcod 21'6 active+clean] repop_all_committed: repop tid 11 all committed 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] eval_repop repgather(0x558e81123e60 21'8 rep_tid=11 committed?=1 r=0) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] commit: repgather(0x558e81123e60 21'8 rep_tid=11 committed?=1 r=0) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] log_op_stats osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 376832 outb 0 lat 0.012503 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] sending reply on osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x558e8402ef00 2019-12-07T19:33:58.792+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(143 smithi13140796-48 [append 0~376832] v21'8 uv8 ondisk = 0) v8 -- 0x558e8402ef00 con 0x558e83d68000 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.792+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] publish_stats_to_osd 21:23 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] removing repgather(0x558e81123e60 21'8 rep_tid=11 committed?=1 r=0) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] q front is repgather(0x558e81123e60 21'8 rep_tid=11 committed?=1 r=0) 2019-12-07T19:33:58.792+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 15 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83feea00 con 0x558e818d6800 2019-12-07T19:33:58.792+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:33:58.792+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0be40 prio 196 cost 0 latency 0.000009 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.792+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] remove_repop repgather(0x558e81123e60 21'8 rep_tid=11 committed?=1 r=0) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] try_finish_rmw: Op(1:4c9e444c:::smithi13140796-48:head v=21'8 tt=0'0 tid=11 reqid=client.4540.0:143 client_op=osd_op(client.4540.0:143 1.2s0 1:4c9e444c:::smithi13140796-48:head [append 0~376832 in=376832b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[425984~376832]}) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] try_finish_rmw: ExtentCache( Extents(1:4c9e444c:::smithi13140796-48:head)[ Extent(425984~376832:8) ) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e81942c60 finish 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0be40 prio 196 cost 0 latency 0.000051 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=12, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] handle_sub_write_reply Calling on_all_commit on Op(1:4c9e444c:::smithi13140796-48:head v=21'9 tt=0'0 tid=12 reqid=client.4540.0:144 client_op=osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'8 crt=21'7 lcod 21'7 mlcod 21'7 active+clean] repop_all_committed: repop tid 12 all committed 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] eval_repop repgather(0x558e8023b7a0 21'9 rep_tid=12 committed?=1 r=0) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] commit: repgather(0x558e8023b7a0 21'9 rep_tid=12 committed?=1 r=0) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] log_op_stats osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 inb 0 outb 0 lat 0.012597 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] sending reply on osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) v8 0x558e83c72f00 2019-12-07T19:33:58.792+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(144 smithi13140796-48 [setxattr (56)] v21'9 uv9 ondisk = 0) v8 -- 0x558e83c72f00 con 0x558e83d68000 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.792+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] publish_stats_to_osd 21:24 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] removing repgather(0x558e8023b7a0 21'9 rep_tid=12 committed?=1 r=0) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] q front is repgather(0x558e8023b7a0 21'9 rep_tid=12 committed?=1 r=0) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] remove_repop repgather(0x558e8023b7a0 21'9 rep_tid=12 committed?=1 r=0) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] requeue_op 0x558e83b0b4a0 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] try_finish_rmw: Op(1:4c9e444c:::smithi13140796-48:head v=21'9 tt=0'0 tid=12 reqid=client.4540.0:144 client_op=osd_op(client.4540.0:144 1.2s0 1:4c9e444c:::smithi13140796-48:head [setxattr _header (56) in=63b] snapc 0=[] ondisk+write+known_if_redirected e21) roll_forward_to=21'7 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) queued 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] try_reads_to_commit: starting commit on Op(1:4c9e444c:::smithi13140796-48:head v=0'0 tt=0'0 tid=13 reqid=client.4540.0:144 roll_forward_to=21'9 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] try_reads_to_commit: written: {} 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] try_reads_to_commit: op: Op(1:4c9e444c:::smithi13140796-48:head v=0'0 tt=0'0 tid=13 reqid=client.4540.0:144 roll_forward_to=21'9 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] try_reads_to_commit: written_set: {} 2019-12-07T19:33:58.792+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 21/18 ECSubWrite(tid=13, reqid=client.4540.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 -- 0x558e83e55800 con 0x558e818d7c00 2019-12-07T19:33:58.792+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 21/18 ECSubWrite(tid=13, reqid=client.4540.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 -- 0x558e83e56000 con 0x558e818d7000 2019-12-07T19:33:58.792+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 21/18 ECSubWrite(tid=13, reqid=client.4540.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 -- 0x558e83e56800 con 0x558e818fe400 2019-12-07T19:33:58.792+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s4 21/18 ECSubWrite(tid=13, reqid=client.4540.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 -- 0x558e83e57000 con 0x558e818d6800 2019-12-07T19:33:58.792+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpWrite(1.2s3 21/18 ECSubWrite(tid=13, reqid=client.4540.0:144, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 -- 0x558e83e57800 con 0x558e818ffc00 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'7 lcod 21'8 mlcod 21'8 active+clean] append_log log((0'0,21'9], crt=21'7) [] 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] rollforward: entry=21'8 (21'7) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:143 2019-12-07T19:33:58.780115+0000 0 ObjectCleanRegions clean_offsets: [0~425984,802816~18446744073708748799], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] rollforward: entry=21'9 (21'8) modify 1:4c9e444c:::smithi13140796-48:head by client.4540.0:144 2019-12-07T19:33:58.780144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] append_log approx pg log length = 9 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] append_log transaction_applied = 1 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.792+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e83c56300 seq 16 2019-12-07T19:33:58.792+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56300 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56300 onodes shared_blobs 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 prepare 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56300 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 io_done 2019-12-07T19:33:58.792+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.792+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0be40 finish 2019-12-07T19:33:58.792+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.792+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.792+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8) prio 63 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.792+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.792+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.792+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.792+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3f6df~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.792+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3f6df 0x3f6df~471 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b4a0 prio 63 cost 0 latency 0.012833 osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.792+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.792+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~10000 x_off 0xf6df 2019-12-07T19:33:58.792+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x6df 2019-12-07T19:33:58.792+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.792+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb50 and padding block with 0x4b0 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] _handle_message: osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] do_op: op osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] do_op osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 may_read -> write-ordered flags ondisk+read+rwordered+known_if_redirected 2019-12-07T19:33:58.792+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] get_object_context: found obc in cache: 0x558e83ff0000 2019-12-07T19:33:58.792+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x3fb50 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] get_object_context: 0x558e83ff0000 1:4c9e444c:::smithi13140796-48:head rwstate(none n=0 w=0) oi: 1:4c9e444c:::smithi13140796-48:head(21'9 client.4540.0:144 dirty|data_digest s 802816 uv 9 dd 18c6bbbb alloc_hint [0 0 0]) exists: 1 ssc: 0x558e81123d40 snapset: 0=[]:{} 2019-12-07T19:33:58.792+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.792+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.792+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.792+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] find_object_context 1:4c9e444c:::smithi13140796-48:head @head oi=1:4c9e444c:::smithi13140796-48:head(21'9 client.4540.0:144 dirty|data_digest s 802816 uv 9 dd 18c6bbbb alloc_hint [0 0 0]) 2019-12-07T19:33:58.792+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000111725s (0.000000082s flush + 0.000111643s kv commit) 2019-12-07T19:33:58.792+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.792+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.792+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56300> 2019-12-07T19:33:58.792+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.792+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 kv_submitted 2019-12-07T19:33:58.792+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56300 2019-12-07T19:33:58.792+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 finishing 2019-12-07T19:33:58.792+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56300 onodes 2019-12-07T19:33:58.792+0000 7fe2c6a64700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] do_op oi 1:4c9e444c:::smithi13140796-48:head(21'9 client.4540.0:144 dirty|data_digest s 802816 uv 9 dd 18c6bbbb alloc_hint [0 0 0]) 2019-12-07T19:33:58.792+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56300 done 2019-12-07T19:33:58.792+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:33:58.792+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56300 [] 2019-12-07T19:33:58.792+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.792+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] do_op obc obc(1:4c9e444c:::smithi13140796-48:head rwstate(excl n=1 w=0)) 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] execute_ctx 0x558e81228e00 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] execute_ctx 1:4c9e444c:::smithi13140796-48:head [read 0~1] ov 21'9 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] do_osd_op 1:4c9e444c:::smithi13140796-48:head [read 0~1] 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] do_osd_op read 0~1 2019-12-07T19:33:58.792+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] do_read 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] async_read noted for 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.792+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] get_all_avail_shards: checking acting 0(0) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] get_all_avail_shards: checking acting 4(1) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] get_all_avail_shards: checking acting 5(4) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] get_all_avail_shards: checking acting 6(3) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] start_read_op: starting ReadOp(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:4c9e444c:::smithi13140796-48:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] do_read_op: starting read ReadOp(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:4c9e444c:::smithi13140796-48:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402fb80 con 0x558e802a5400 2019-12-07T19:33:58.793+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.2s2 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402f180 con 0x558e818d7c00 2019-12-07T19:33:58.793+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.2s1 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402e780 con 0x558e818fe400 2019-12-07T19:33:58.793+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558e8402fb80 con 0x558e802a5400 2019-12-07T19:33:58.793+0000 7fe2d427f700 15 osd.0 21 enqueue_op 0x558e81943e40 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.793+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpRead(1.2s3 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402ec80 con 0x558e818ffc00 2019-12-07T19:33:58.793+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] do_read_op: started ReadOp(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:4c9e444c:::smithi13140796-48:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, []))}, priority=127, obj_to_source={1:4c9e444c:::smithi13140796-48:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:4c9e444c:::smithi13140796-48:head,1(2)=1:4c9e444c:::smithi13140796-48:head,4(1)=1:4c9e444c:::smithi13140796-48:head,6(3)=1:4c9e444c:::smithi13140796-48:head}, in_progress=0(0),1(2),4(1),6(3)) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83b0b4a0 finish 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.793+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 16 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83fef200 con 0x558e818d6800 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.793+0000 7fe2e753b700 15 osd.0 21 enqueue_op 0x558e83b0bce0 prio 196 cost 0 latency 0.000013 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.793+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943e40 prio 127 cost 0 latency 0.000091 MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 80 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028e00 con 0x558e818d7c00 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83edc000 prio 196 cost 0 latency 0.000015 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 19 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028400 con 0x558e818ffc00 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83edc160 prio 196 cost 0 latency 0.000012 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpRead(1.2s0 21/18 ECSubRead(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=0,4096,0}, subchunks={1:4c9e444c:::smithi13140796-48:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 44 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818eee00 con 0x558e818d7000 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83edc2c0 prio 196 cost 0 latency 0.000008 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.793+0000 7fe2c6a64700 25 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 0x0~1000 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x31000 (200704) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83edee70 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edefc0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83edee70 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edefc0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83edee70 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edefc0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83edefd8 in 0x558e802195e0) _discard 0x0~1000 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 _add level 1 near 0 on buffer(0x558e83d118c0 space 0x558e83edefd8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 0x0~1000 = 4096 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] get_hash_info: Getting attr on 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] get_hash_info: not in cache 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] get_hash_info: found on disk, size 200704 2019-12-07T19:33:58.793+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# hinfo_key 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# hinfo_key = 0 2019-12-07T19:33:58.793+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 -- 0x558e81a03680 con 0x558e802a5400 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e81943e40 finish 2019-12-07T19:33:58.793+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558e81a03680 con 0x558e802a5400 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2d427f700 15 osd.0 21 enqueue_op 0x558e83edd1e0 prio 127 cost 0 latency 0.000013 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.793+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0bce0 prio 196 cost 0 latency 0.000367 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0bce0 finish 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83edc000 prio 196 cost 0 latency 0.000351 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83edc000 finish 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83edc2c0 prio 196 cost 0 latency 0.000303 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83edc2c0 finish 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83edc160 prio 196 cost 0 latency 0.000399 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83edc160 finish 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83edd1e0 prio 127 cost 0 latency 0.000171 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 81 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 ==== 4261+0+0 (unknown 0 0 0) 0x558e8402ec80 con 0x558e818d7c00 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83edc420 prio 127 cost 0 latency 0.000016 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=14, attrs_read=0) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 20 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 ==== 4261+0+0 (unknown 0 0 0) 0x558e8402f180 con 0x558e818ffc00 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83edc580 prio 127 cost 0 latency 0.000010 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.793+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:4c9e444c:::smithi13140796-48:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [0(0),4096]))}, priority=127, obj_to_source={1:4c9e444c:::smithi13140796-48:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:4c9e444c:::smithi13140796-48:head,1(2)=1:4c9e444c:::smithi13140796-48:head,4(1)=1:4c9e444c:::smithi13140796-48:head,6(3)=1:4c9e444c:::smithi13140796-48:head}, in_progress=1(2),4(1),6(3)) 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83edd1e0 finish 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83edc420 prio 127 cost 0 latency 0.000059 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=14, attrs_read=0) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:4c9e444c:::smithi13140796-48:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [0(0),4096, 1(2),4096]))}, priority=127, obj_to_source={1:4c9e444c:::smithi13140796-48:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:4c9e444c:::smithi13140796-48:head,1(2)=1:4c9e444c:::smithi13140796-48:head,4(1)=1:4c9e444c:::smithi13140796-48:head,6(3)=1:4c9e444c:::smithi13140796-48:head}, in_progress=4(1),6(3)) 2019-12-07T19:33:58.793+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e83edc420 finish 2019-12-07T19:33:58.793+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83edc580 prio 127 cost 0 latency 0.000081 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=14, attrs_read=0) 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_sub_read_reply readop not complete: ReadOp(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:4c9e444c:::smithi13140796-48:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [0(0),4096, 1(2),4096, 6(3),4096]))}, priority=127, obj_to_source={1:4c9e444c:::smithi13140796-48:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:4c9e444c:::smithi13140796-48:head,1(2)=1:4c9e444c:::smithi13140796-48:head,4(1)=1:4c9e444c:::smithi13140796-48:head,6(3)=1:4c9e444c:::smithi13140796-48:head}, in_progress=4(1)) 2019-12-07T19:33:58.793+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83edc580 finish 2019-12-07T19:33:58.793+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.794+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 34 ==== MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 ==== 4261+0+0 (unknown 0 0 0) 0x558e8402e780 con 0x558e818fe400 2019-12-07T19:33:58.794+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e818058c0 prio 127 cost 0 latency 0.000013 epoch 21 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.794+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.794+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.794+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 35 ==== MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83fee200 con 0x558e818fe400 2019-12-07T19:33:58.794+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.794+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.794+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2) prio 127 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.794+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a160 prio 196 cost 0 latency 0.000016 epoch 21 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.794+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) 2019-12-07T19:33:58.794+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e818058c0 prio 127 cost 0 latency 0.000034 MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.794+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.794+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) queued 2019-12-07T19:33:58.794+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpReadReply(1.2s0 21/18 ECSubReadReply(tid=14, attrs_read=0)) v2 2019-12-07T19:33:58.794+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_sub_read_reply: reply ECSubReadReply(tid=14, attrs_read=0) 2019-12-07T19:33:58.794+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_sub_read_reply have shard=0 2019-12-07T19:33:58.794+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_sub_read_reply have shard=2 2019-12-07T19:33:58.794+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_sub_read_reply have shard=1 2019-12-07T19:33:58.794+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_sub_read_reply have shard=3 2019-12-07T19:33:58.794+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_sub_read_reply Complete: ReadOp(tid=14, to_read={1:4c9e444c:::smithi13140796-48:head=read_request_t(to_read=[0,16384,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],6(3)=[0,8]}, want_attrs=0)}, complete={1:4c9e444c:::smithi13140796-48:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 16384, [0(0),4096, 1(2),4096, 4(1),4096, 6(3),4096]))}, priority=127, obj_to_source={1:4c9e444c:::smithi13140796-48:head=0(0),1(2),4(1),6(3)}, source_to_obj={0(0)=1:4c9e444c:::smithi13140796-48:head,1(2)=1:4c9e444c:::smithi13140796-48:head,4(1)=1:4c9e444c:::smithi13140796-48:head,6(3)=1:4c9e444c:::smithi13140796-48:head}, in_progress=) 2019-12-07T19:33:58.794+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] objects_read_async_cb: got: {1:4c9e444c:::smithi13140796-48:head=0,{0~16384(16384)}} 2019-12-07T19:33:58.794+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:33:58.794+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] execute_ctx 0x558e81228e00 2019-12-07T19:33:58.794+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] execute_ctx 1:4c9e444c:::smithi13140796-48:head [read 0~1 out=1b] ov 21'9 2019-12-07T19:33:58.794+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] do_osd_op 1:4c9e444c:::smithi13140796-48:head [read 0~1 out=1b] 2019-12-07T19:33:58.794+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] do_osd_op read 0~1 out=1b 2019-12-07T19:33:58.794+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] execute_ctx alloc reply 0x558e8402ec80 result 0 2019-12-07T19:33:58.794+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:33:58.794+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] log_op_stats osd_op(client.4540.0:145 1.2s0 1:4c9e444c:::smithi13140796-48:head [read 0~1 out=1b] snapc 0=[] ondisk+read+rwordered+known_if_redirected e21) v8 inb 0 outb 1 lat 0.014248 2019-12-07T19:33:58.794+0000 7fe2c6a64700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:58.794+0000 7fe2c6a64700 15 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] publish_stats_to_osd 21:25 2019-12-07T19:33:58.794+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(145 smithi13140796-48 [read 0~1 out=1b] v0'0 uv9 ondisk = 0) v8 -- 0x558e8402ec80 con 0x558e83d68000 2019-12-07T19:33:58.794+0000 7fe2c6a64700 10 osd.0 21 dequeue_op 0x558e818058c0 finish 2019-12-07T19:33:58.794+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.794+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2) prio 196 cost 0 e21) pg 0x558e81075000 2019-12-07T19:33:58.794+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.794+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a160 prio 196 cost 0 latency 0.000222 MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:33:58.794+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] _handle_message: MOSDECSubOpWriteReply(1.2s0 21/18 ECSubWriteReply(tid=13, last_complete=21'9, committed=1, applied=1)) v2 2019-12-07T19:33:58.794+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] try_finish_rmw: Op(1:4c9e444c:::smithi13140796-48:head v=0'0 tt=0'0 tid=13 reqid=client.4540.0:144 roll_forward_to=21'9 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:33:58.794+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] try_finish_rmw: ExtentCache( ) 2019-12-07T19:33:58.794+0000 7fe2caa6c700 20 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:33:58.794+0000 7fe2caa6c700 10 osd.0 21 dequeue_op 0x558e83b0a160 finish 2019-12-07T19:33:58.794+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:58.794+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 36 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=22, reqid=client.4540.0:146, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 112564+0+0 (unknown 0 0 0) 0x558e83e56800 con 0x558e818fe400 2019-12-07T19:33:58.794+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a6e0 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=22, reqid=client.4540.0:146, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.794+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=22, reqid=client.4540.0:146, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=22, reqid=client.4540.0:146, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=22, reqid=client.4540.0:146, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.794+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e83b0a6e0 prio 127 cost 0 latency 0.000062 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=22, reqid=client.4540.0:146, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.794+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=22, reqid=client.4540.0:146, at_version=21'5, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.794+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'4 (0'0,21'4] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'4], crt=21'4) [21'5 (0'0) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:146 2019-12-07T19:33:58.783523+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.794+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] add_log_entry 21'5 (0'0) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:146 2019-12-07T19:33:58.783523+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.794+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log approx pg log length = 5 2019-12-07T19:33:58.794+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.794+0000 7fe2ca26b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.794+0000 7fe2ca26b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'5, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.794+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e8195e600 seq 13 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs3_head 0x558e819c41e0) get_onode oid 3#1:d313ddd1:::smithi13140796-49:head# key 0x838000000000000001d313ddd121736d'ithi13140796-49!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.794+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1105 2019-12-07T19:33:58.794+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# = 0 2019-12-07T19:33:58.794+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 2 keys 2019-12-07T19:33:58.794+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 2 keys = 0 2019-12-07T19:33:58.794+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.794+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.794+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~1b000 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:d313ddd1:::smithi13140796-49:head# 0x0~1b000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.794+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1b000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8195e600 14 blobs 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.794+0000 7fe2ca26b700 10 fbmap_alloc 0x558e80233300 allocate 0x1b000/1000,1b000,0 2019-12-07T19:33:58.794+0000 7fe2ca26b700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x2a4000~1b000/1000,1b000,0 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x2a4000~1b000] 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194e700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e9ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194e700 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9ce0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194e700) get_ref 0x0~2000 Blob(0x558e8194e700 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9ce0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194e700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e8194e700 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9ce0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e817e9cf8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194ecb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194ee00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194ecb0 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194ee00 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194ecb0) get_ref 0x0~2000 Blob(0x558e8194ecb0 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194ee00 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194ecb0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e8194ecb0 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ee00 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194ee18 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194ed20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194eee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194ed20 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194eee0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194ed20) get_ref 0x0~2000 Blob(0x558e8194ed20 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194eee0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194ed20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e8194ed20 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194eee0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194eef8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194ef50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c64ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194ef50 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c64ee0 sbid 0x0)) 2019-12-07T19:33:58.794+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194ef50) get_ref 0x0~2000 Blob(0x558e8194ef50 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c64ee0 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194ef50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e8194ef50 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64ee0 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83c64ef8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0fd50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194fab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0fd50 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194fab0 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d0fd50) get_ref 0x0~2000 Blob(0x558e83d0fd50 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194fab0 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d0fd50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83d0fd50 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fab0 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194fac8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194fb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194f9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194fb20 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f9d0 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194fb20) get_ref 0x0~2000 Blob(0x558e8194fb20 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f9d0 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194fb20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e8194fb20 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f9d0 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194f9e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194e7e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d1880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194e7e0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d1880 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194e7e0) get_ref 0x0~2000 Blob(0x558e8194e7e0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d1880 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194e7e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e8194e7e0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d1880 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e811d1898 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e811d09a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81867490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e811d09a0 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81867490 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e811d09a0) get_ref 0x0~2000 Blob(0x558e811d09a0 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81867490 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e811d09a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e811d09a0 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867490 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e818674a8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d02460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d024d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d02460 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d024d0 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d02460) get_ref 0x0~2000 Blob(0x558e83d02460 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d024d0 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d02460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83d02460 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d024d0 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83d024e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd2000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd2000 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5c00 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e83dd2000) get_ref 0x0~2000 Blob(0x558e83dd2000 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5c00 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e83dd2000) get_ref init 0x2000, 1000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83dd2000 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5c00 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5c18 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac48c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac48c0 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5c70 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac48c0) get_ref 0x0~2000 Blob(0x558e83ac48c0 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5c70 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac48c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83ac48c0 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5c70 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5c88 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e9110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c64c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e9110 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c64c40 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e817e9110) get_ref 0x0~2000 Blob(0x558e817e9110 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c64c40 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e817e9110) get_ref init 0x2000, 1000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e817e9110 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64c40 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83c64c58 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83c64cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83c650a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83c64cb0 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c650a0 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e83c64cb0) get_ref 0x0~2000 Blob(0x558e83c64cb0 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c650a0 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e83c64cb0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83c64cb0 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c650a0 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83c650b8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84014d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84014e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84014d90 blob([0x2be000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84014e00 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e84014d90) get_ref 0x0~1000 Blob(0x558e84014d90 blob([0x2be000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84014e00 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.blob(0x558e84014d90) get_ref init 0x1000, 1000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~1000: 0x0~1000 Blob(0x558e84014d90 blob([0x2be000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e84014e00 sbid 0x0)) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84014e18 in 0x558e80219c00) _discard 0x0~1000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1b000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e8402f008) dirty_range mark inline shard dirty 2019-12-07T19:33:58.795+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~1b000 = 0 2019-12-07T19:33:58.795+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.795+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.795+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.795+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e600 cost 172214 (15 ios * 4000 + 112214 bytes) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e600 onodes 0x558e8402ef00 shared_blobs 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e8402f008) update 3#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e8402f008) update inline shard 266 bytes from 14 extents 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:d313ddd1:::smithi13140796-49:head# is 685 (413 bytes onode + 2 bytes spanning blobs + 270 bytes inline extents) 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e600 allocated 0x[2a4000~1b000] released 0x[] 2019-12-07T19:33:58.795+0000 7fe2ca26b700 10 freelist allocate 0x2a4000~1b000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:33:58.795+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 prepare 2019-12-07T19:33:58.795+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8195e600 2019-12-07T19:33:58.795+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'4 crt=21'4 lcod 21'4 active mbc={}] op_applied version 21'5 2019-12-07T19:33:58.795+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e83b0a6e0 finish 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.795+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 aio_wait 2019-12-07T19:33:58.795+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e600 2019-12-07T19:33:58.795+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 io_done 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1200 < min_flush_size 524288 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1408 < min_flush_size 524288 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ef00 had 1 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1431 < min_flush_size 524288 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1431 < min_flush_size 524288 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x3fb50~597 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x3fb50 0x3fb50~597 to file(ino 16 size 0x40000 mtime 2019-12-07T19:33:58.696378+0000 allocated 40000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~10000]) 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _allocate len 0xe7 from 0 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _allocate len 0xe7 from 1 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd170000 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd170000~10000/10000,10000,acd170000 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x50000 to include allocated 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 26 (was clean) 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0xfb50 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb50 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xe7 and padding block with 0xf19 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x50000 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x400e7 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (26) on file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]), flushing log 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 26 len 0x49 crc 0xaaa75b30) 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x19000~1000 to file(ino 1 size 0x19000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x19000 0x19000~1000 to file(ino 1 size 0x19000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x1a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x19000 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x1a000 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:33:58.795+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000017 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 26 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000347517s (0.000035712s flush + 0.000311805s kv commit) 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.795+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e600> 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.795+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 kv_submitted 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e600 2019-12-07T19:33:58.795+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 finishing 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e600 onodes 0x558e8402ef00 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d1898 in 0x558e80219c00) _finish_write discard buffer(0x558e83d10900 space 0x558e811d1898 0x0~2000 writing nocache) 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e9cf8 in 0x558e80219c00) _finish_write discard buffer(0x558e811701b0 space 0x558e817e9cf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.795+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e818674a8 in 0x558e80219c00) _finish_write discard buffer(0x558e83d10990 space 0x558e818674a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194ee18 in 0x558e80219c00) _finish_write discard buffer(0x558e81171050 space 0x558e8194ee18 0x0~2000 writing nocache) 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194eef8 in 0x558e80219c00) _finish_write discard buffer(0x558e83d119e0 space 0x558e8194eef8 0x0~2000 writing nocache) 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194f9e8 in 0x558e80219c00) _finish_write discard buffer(0x558e83d105a0 space 0x558e8194f9e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194fac8 in 0x558e80219c00) _finish_write discard buffer(0x558e802299e0 space 0x558e8194fac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac5c18 in 0x558e80219c00) _finish_write discard buffer(0x558e83d10cf0 space 0x558e83ac5c18 0x0~2000 writing nocache) 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac5c88 in 0x558e80219c00) _finish_write discard buffer(0x558e83d103f0 space 0x558e83ac5c88 0x0~2000 writing nocache) 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c64c58 in 0x558e80219c00) _finish_write discard buffer(0x558e83d11e60 space 0x558e83c64c58 0x0~2000 writing nocache) 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c64ef8 in 0x558e80219c00) _finish_write discard buffer(0x558e83d11a70 space 0x558e83c64ef8 0x0~2000 writing nocache) 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83c650b8 in 0x558e80219c00) _finish_write discard buffer(0x558e83d11dd0 space 0x558e83c650b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d024e8 in 0x558e80219c00) _finish_write discard buffer(0x558e83d11560 space 0x558e83d024e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84014e18 in 0x558e80219c00) _finish_write discard buffer(0x558e83d11d40 space 0x558e84014e18 0x0~1000 writing nocache) 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e600 done 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:58.795+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e600 [] 2019-12-07T19:33:58.795+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.795+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=22, last_complete=21'5, committed=1, applied=1)) v2 -- 0x558e818a9e00 con 0x558e818fe400 2019-12-07T19:33:58.795+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.795+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.796+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 37 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=23, reqid=client.4540.0:150, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 ==== 116660+0+0 (unknown 0 0 0) 0x558e83e54000 con 0x558e818fe400 2019-12-07T19:33:58.796+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a000 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=23, reqid=client.4540.0:150, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.796+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=23, reqid=client.4540.0:150, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=23, reqid=client.4540.0:150, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=23, reqid=client.4540.0:150, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.796+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e83b0a000 prio 127 cost 0 latency 0.000051 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=23, reqid=client.4540.0:150, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 pg pg[1.bs3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.796+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=23, reqid=client.4540.0:150, at_version=21'6, trim_to=0'0, roll_forward_to=21'4)) v2 2019-12-07T19:33:58.796+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'5 (0'0,21'5] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'5], crt=21'4) [21'6 (0'0) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:150 2019-12-07T19:33:58.789897+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.796+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] add_log_entry 21'6 (0'0) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:150 2019-12-07T19:33:58.789897+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.796+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] append_log approx pg log length = 6 2019-12-07T19:33:58.796+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.796+0000 7fe2c6263700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.796+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.796+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e811fb800 seq 14 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs3_head 0x558e819c41e0) get_onode oid 3#1:dce63cc3:::smithi13140796-50:head# key 0x838000000000000001dce63cc321736d'ithi13140796-50!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:33:58.796+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1106 2019-12-07T19:33:58.796+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# = 0 2019-12-07T19:33:58.796+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 2 keys 2019-12-07T19:33:58.796+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 2 keys = 0 2019-12-07T19:33:58.796+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.796+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.796+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 0x0~1c000 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:dce63cc3:::smithi13140796-50:head# 0x0~1c000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.796+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1c000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fb800 14 blobs 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.796+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate 0x1c000/1000,1c000,0 2019-12-07T19:33:58.796+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x2bf000~1c000/1000,1c000,0 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x2bf000~1c000] 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d02620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f48e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.796+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d02620 blob([0x2bf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48e00 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e83d02620) get_ref 0x0~2000 Blob(0x558e83d02620 blob([0x2bf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48e00 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e83d02620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83d02620 blob([0x2bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48e00 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83f48e18 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d37030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d37030 blob([0x2c1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4bd0 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e83d37030) get_ref 0x0~2000 Blob(0x558e83d37030 blob([0x2c1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4bd0 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e83d37030) get_ref init 0x2000, 1000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83d37030 blob([0x2c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4bd0 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ac4be8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0fb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0fb20 blob([0x2c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fb90 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0fb20) get_ref 0x0~2000 Blob(0x558e83d0fb20 blob([0x2c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fb90 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0fb20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83d0fb20 blob([0x2c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fb90 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0fba8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0fc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0fc00 blob([0x2c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fc70 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0fc00) get_ref 0x0~2000 Blob(0x558e83d0fc00 blob([0x2c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fc70 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0fc00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83d0fc00 blob([0x2c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fc70 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0fc88 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0fce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84015490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0fce0 blob([0x2c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015490 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0fce0) get_ref 0x0~2000 Blob(0x558e83d0fce0 blob([0x2c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015490 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0fce0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83d0fce0 blob([0x2c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015490 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e840154a8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84015500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84015570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84015500 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015570 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e84015500) get_ref 0x0~2000 Blob(0x558e84015500 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015570 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e84015500) get_ref init 0x2000, 1000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e84015500 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015570 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015588 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e840155e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84015650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e840155e0 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015650 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e840155e0) get_ref 0x0~2000 Blob(0x558e840155e0 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015650 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e840155e0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e840155e0 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015650 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015668 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e840156c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84015730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e840156c0 blob([0x2cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015730 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e840156c0) get_ref 0x0~2000 Blob(0x558e840156c0 blob([0x2cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015730 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e840156c0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e840156c0 blob([0x2cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015730 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015748 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e840157a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84015810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e840157a0 blob([0x2cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015810 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e840157a0) get_ref 0x0~2000 Blob(0x558e840157a0 blob([0x2cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015810 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e840157a0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e840157a0 blob([0x2cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015810 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015828 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84015880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e840158f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84015880 blob([0x2d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840158f0 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e84015880) get_ref 0x0~2000 Blob(0x558e84015880 blob([0x2d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840158f0 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e84015880) get_ref init 0x2000, 1000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e84015880 blob([0x2d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840158f0 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015908 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84015960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e840159d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84015960 blob([0x2d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840159d0 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e84015960) get_ref 0x0~2000 Blob(0x558e84015960 blob([0x2d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840159d0 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e84015960) get_ref init 0x2000, 1000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e84015960 blob([0x2d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840159d0 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e840159e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84015a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84015ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84015a40 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015ab0 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e84015a40) get_ref 0x0~2000 Blob(0x558e84015a40 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015ab0 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e84015a40) get_ref init 0x2000, 1000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e84015a40 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015ab0 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015ac8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84015b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84015b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84015b20 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015b90 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e84015b20) get_ref 0x0~2000 Blob(0x558e84015b20 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015b90 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e84015b20) get_ref init 0x2000, 1000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e84015b20 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015b90 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015ba8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84015c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84015c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84015c00 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015c70 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e84015c00) get_ref 0x0~2000 Blob(0x558e84015c00 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015c70 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.blob(0x558e84015c00) get_ref init 0x2000, 1000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e84015c00 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015c70 sbid 0x0)) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015c88 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1c000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83cb2388) dirty_range mark inline shard dirty 2019-12-07T19:33:58.797+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 0x0~1c000 = 0 2019-12-07T19:33:58.797+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.797+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.797+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.797+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.797+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fb800 cost 176310 (15 ios * 4000 + 116310 bytes) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fb800 onodes 0x558e83cb2280 shared_blobs 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83cb2388) update 3#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83cb2388) update inline shard 269 bytes from 14 extents 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:dce63cc3:::smithi13140796-50:head# is 688 (413 bytes onode + 2 bytes spanning blobs + 273 bytes inline extents) 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fb800 allocated 0x[2bf000~1c000] released 0x[] 2019-12-07T19:33:58.797+0000 7fe2c6263700 10 freelist allocate 0x2bf000~1c000 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:33:58.797+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 prepare 2019-12-07T19:33:58.797+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e811fb800 2019-12-07T19:33:58.797+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'5 crt=21'4 lcod 21'5 active mbc={}] op_applied version 21'6 2019-12-07T19:33:58.797+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e83b0a000 finish 2019-12-07T19:33:58.797+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.798+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 aio_wait 2019-12-07T19:33:58.798+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fb800 2019-12-07T19:33:58.798+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 io_done 2019-12-07T19:33:58.798+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.798+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.798+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.798+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1400 < min_flush_size 524288 2019-12-07T19:33:58.798+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2280 had 1 2019-12-07T19:33:58.798+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.798+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1423 < min_flush_size 524288 2019-12-07T19:33:58.798+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1423 < min_flush_size 524288 2019-12-07T19:33:58.798+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.798+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x400e7~58f to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.798+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x400e7 0x400e7~58f to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.798+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.798+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x100e7 2019-12-07T19:33:58.798+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xe7 2019-12-07T19:33:58.798+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.798+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x676 and padding block with 0x98a 2019-12-07T19:33:58.798+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x40676 2019-12-07T19:33:58.798+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.798+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.798+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.798+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.798+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000133478s (0.000039228s flush + 0.000094250s kv commit) 2019-12-07T19:33:58.798+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fb800> 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.798+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 kv_submitted 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fb800 2019-12-07T19:33:58.798+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 finishing 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fb800 onodes 0x558e83cb2280 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac4be8 in 0x558e80219c00) _finish_write discard buffer(0x558e83d115f0 space 0x558e83ac4be8 0x0~2000 writing nocache) 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0fba8 in 0x558e80219c00) _finish_write discard buffer(0x558e83d11170 space 0x558e83d0fba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0fc88 in 0x558e80219c00) _finish_write discard buffer(0x558e83d11200 space 0x558e83d0fc88 0x0~2000 writing nocache) 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f48e18 in 0x558e80219c00) _finish_write discard buffer(0x558e83d11440 space 0x558e83f48e18 0x0~2000 writing nocache) 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e840154a8 in 0x558e80219c00) _finish_write discard buffer(0x558e83d11320 space 0x558e840154a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84015588 in 0x558e80219c00) _finish_write discard buffer(0x558e83d10360 space 0x558e84015588 0x0~2000 writing nocache) 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84015668 in 0x558e80219c00) _finish_write discard buffer(0x558e83d10510 space 0x558e84015668 0x0~2000 writing nocache) 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84015748 in 0x558e80219c00) _finish_write discard buffer(0x558e84032a20 space 0x558e84015748 0x0~2000 writing nocache) 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84015828 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8f170 space 0x558e84015828 0x0~2000 writing nocache) 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84015908 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8fc20 space 0x558e84015908 0x0~2000 writing nocache) 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e840159e8 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8f290 space 0x558e840159e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84015ac8 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8e990 space 0x558e84015ac8 0x0~2000 writing nocache) 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84015ba8 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8eab0 space 0x558e84015ba8 0x0~2000 writing nocache) 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84015c88 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8ecf0 space 0x558e84015c88 0x0~2000 writing nocache) 2019-12-07T19:33:58.798+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=23, last_complete=21'6, committed=1, applied=1)) v2 -- 0x558e84028200 con 0x558e818fe400 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb800 done 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:58.798+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fb800 [] 2019-12-07T19:33:58.798+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.798+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.798+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.798+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.799+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 38 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=25, reqid=client.4540.0:147, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 ==== 96536+0+0 (unknown 0 0 0) 0x558e819cf800 con 0x558e818fe400 2019-12-07T19:33:58.799+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0ba20 prio 127 cost 0 latency 0.000021 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=25, reqid=client.4540.0:147, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.799+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=25, reqid=client.4540.0:147, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=25, reqid=client.4540.0:147, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=25, reqid=client.4540.0:147, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e83b0ba20 prio 127 cost 0 latency 0.000058 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=25, reqid=client.4540.0:147, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 pg pg[1.bs3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=25, reqid=client.4540.0:147, at_version=21'7, trim_to=0'0, roll_forward_to=21'5)) v2 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'4 active mbc={}] append_log log((0'0,21'6], crt=21'4) [21'7 (21'5) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:147 2019-12-07T19:33:58.786362+0000 0 ObjectCleanRegions clean_offsets: [0~442368,819200~18446744073708732415], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'4 lcod 21'6 active mbc={}] add_log_entry 21'7 (21'5) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:147 2019-12-07T19:33:58.786362+0000 0 ObjectCleanRegions clean_offsets: [0~442368,819200~18446744073708732415], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] rollforward: entry=21'5 (0'0) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:146 2019-12-07T19:33:58.783523+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] append_log approx pg log length = 7 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.800+0000 7fe2ca26b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e8195e900 seq 15 2019-12-07T19:33:58.800+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 2 keys 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 2 keys = 0 2019-12-07T19:33:58.800+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.800+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x1b000~17000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:d313ddd1:::smithi13140796-49:head# 0x1b000~17000 - have 0x1b000 (110592) bytes fadvise_flags 0x0 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x1b000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8195e900 12 blobs 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 fbmap_alloc 0x558e80233300 allocate 0x17000/1000,17000,0 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x2db000~17000/1000,17000,0 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x2db000~17000] 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e840152d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84015260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e840152d0 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015260 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e840152d0) get_ref 0x0~2000 Blob(0x558e840152d0 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84015260 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e840152d0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1b000~2000: 0x0~2000 Blob(0x558e840152d0 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015260 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015278 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84015340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e840153b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84015340 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840153b0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015340) get_ref 0x0~2000 Blob(0x558e84015340 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840153b0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015340) get_ref init 0x2000, 1000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1d000~2000: 0x0~2000 Blob(0x558e84015340 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840153b0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e840153c8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6c770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6c770 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c7e0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c770) get_ref 0x0~2000 Blob(0x558e83e6c770 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c7e0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c770) get_ref init 0x2000, 1000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1f000~2000: 0x0~2000 Blob(0x558e83e6c770 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c7e0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c7f8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6c850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6c850 blob([0x2e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c8c0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c850) get_ref 0x0~2000 Blob(0x558e83e6c850 blob([0x2e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c8c0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c850) get_ref init 0x2000, 1000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x21000~2000: 0x0~2000 Blob(0x558e83e6c850 blob([0x2e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c8c0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c8d8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6c930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6c930 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c9a0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c930) get_ref 0x0~2000 Blob(0x558e83e6c930 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c9a0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c930) get_ref init 0x2000, 1000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x23000~2000: 0x0~2000 Blob(0x558e83e6c930 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c9a0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c9b8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6ca10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6ca80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6ca10 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6ca80 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6ca10) get_ref 0x0~2000 Blob(0x558e83e6ca10 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6ca80 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6ca10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x25000~2000: 0x0~2000 Blob(0x558e83e6ca10 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6ca80 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6ca98 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6caf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e8af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6caf0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8af0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6caf0) get_ref 0x0~2000 Blob(0x558e83e6caf0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8af0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6caf0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x27000~2000: 0x0~2000 Blob(0x558e83e6caf0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8af0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e817e8b08 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e9d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e840151f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e9d50 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840151f0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e817e9d50) get_ref 0x0~2000 Blob(0x558e817e9d50 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840151f0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e817e9d50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x29000~2000: 0x0~2000 Blob(0x558e817e9d50 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840151f0 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015208 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84015180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6cb60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84015180 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6cb60 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015180) get_ref 0x0~2000 Blob(0x558e84015180 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6cb60 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015180) get_ref init 0x2000, 1000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2b000~2000: 0x0~2000 Blob(0x558e84015180 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cb60 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6cb78 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6cbd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6cc40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6cbd0 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6cc40 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6cbd0) get_ref 0x0~2000 Blob(0x558e83e6cbd0 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6cc40 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6cbd0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2d000~2000: 0x0~2000 Blob(0x558e83e6cbd0 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cc40 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6cc58 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6ccb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6cd20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6ccb0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6cd20 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6ccb0) get_ref 0x0~2000 Blob(0x558e83e6ccb0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6cd20 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6ccb0) get_ref init 0x2000, 1000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2f000~2000: 0x0~2000 Blob(0x558e83e6ccb0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cd20 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6cd38 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6cd90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6ce00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6cd90 blob([!~1000,0x2f1000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6ce00 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6cd90) get_ref 0x1000~1000 Blob(0x558e83e6cd90 blob([!~1000,0x2f1000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6ce00 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6cd90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x31000~1000: 0x1000~1000 Blob(0x558e83e6cd90 blob([!~1000,0x2f1000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558e83e6ce00 sbid 0x0)) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6ce18 in 0x558e80219c00) _discard 0x1000~1000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x32000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e8402f008) dirty_range mark inline shard dirty 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x1b000~17000 = 0 2019-12-07T19:33:58.800+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.800+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e900 cost 148186 (13 ios * 4000 + 96186 bytes) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e900 onodes 0x558e8402ef00 shared_blobs 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e8402f008) update 3#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e8402f008) update inline shard 508 bytes from 26 extents 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:d313ddd1:::smithi13140796-49:head# is 927 (413 bytes onode + 2 bytes spanning blobs + 512 bytes inline extents) 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e900 allocated 0x[2db000~17000] released 0x[] 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 freelist allocate 0x2db000~17000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 prepare 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8195e900 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'6 crt=21'5 lcod 21'6 active mbc={}] op_applied version 21'7 2019-12-07T19:33:58.800+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e83b0ba20 finish 2019-12-07T19:33:58.800+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.800+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 aio_wait 2019-12-07T19:33:58.800+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e900 2019-12-07T19:33:58.800+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 io_done 2019-12-07T19:33:58.800+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.800+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.800+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.800+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2067 < min_flush_size 524288 2019-12-07T19:33:58.800+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ef00 had 1 2019-12-07T19:33:58.800+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.800+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2090 < min_flush_size 524288 2019-12-07T19:33:58.800+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2090 < min_flush_size 524288 2019-12-07T19:33:58.800+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.800+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x40676~82a to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.800+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x40676 0x40676~82a to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.800+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.800+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x10676 2019-12-07T19:33:58.800+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x676 2019-12-07T19:33:58.800+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.800+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xea0 and padding block with 0x160 2019-12-07T19:33:58.801+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x40ea0 2019-12-07T19:33:58.801+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.801+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.801+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.801+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.801+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000213966s (0.000039167s flush + 0.000174799s kv commit) 2019-12-07T19:33:58.801+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e900> 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.801+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 kv_submitted 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e900 2019-12-07T19:33:58.801+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 finishing 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e900 onodes 0x558e8402ef00 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e8b08 in 0x558e80219c00) _finish_write discard buffer(0x558e83d10990 space 0x558e817e8b08 0x0~2000 writing nocache) 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6c7f8 in 0x558e80219c00) _finish_write discard buffer(0x558e83d10090 space 0x558e83e6c7f8 0x0~2000 writing nocache) 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6c8d8 in 0x558e80219c00) _finish_write discard buffer(0x558e83d11b90 space 0x558e83e6c8d8 0x0~2000 writing nocache) 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6c9b8 in 0x558e80219c00) _finish_write discard buffer(0x558e83d11c20 space 0x558e83e6c9b8 0x0~2000 writing nocache) 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6ca98 in 0x558e80219c00) _finish_write discard buffer(0x558e83d10900 space 0x558e83e6ca98 0x0~2000 writing nocache) 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6cb78 in 0x558e80219c00) _finish_write discard buffer(0x558e811701b0 space 0x558e83e6cb78 0x0~2000 writing nocache) 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6cc58 in 0x558e80219c00) _finish_write discard buffer(0x558e81171050 space 0x558e83e6cc58 0x0~2000 writing nocache) 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6cd38 in 0x558e80219c00) _finish_write discard buffer(0x558e83d119e0 space 0x558e83e6cd38 0x0~2000 writing nocache) 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6ce18 in 0x558e80219c00) _finish_write discard buffer(0x558e83d105a0 space 0x558e83e6ce18 0x1000~1000 writing nocache) 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84015208 in 0x558e80219c00) _finish_write discard buffer(0x558e802299e0 space 0x558e84015208 0x0~2000 writing nocache) 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84015278 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8e630 space 0x558e84015278 0x0~2000 writing nocache) 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e840153c8 in 0x558e80219c00) _finish_write discard buffer(0x558e83d11ef0 space 0x558e840153c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e900 done 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:58.801+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e900 [] 2019-12-07T19:33:58.801+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.801+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.801+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 82 ==== MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=81, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8402e280 con 0x558e818d7c00 2019-12-07T19:33:58.801+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83edc840 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=81, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.801+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=81, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=81, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=81, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.801+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e83edc840 prio 127 cost 0 latency 0.000061 MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=81, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.801+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.801+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=81, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.801+0000 7fe2c8267700 25 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.801+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 0x0~19000 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~19000 size 0x19000 (102400) 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f7a0 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac59d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4a80 blob([0xe4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9a40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9b90 blob([0xe6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9c00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9c70 blob([0xe8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9ce0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9d50 blob([0xea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9dc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9e30 blob([0xec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9ea0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9f10 blob([0xee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9f80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e000 blob([0xf0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e070 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e0e0 blob([0xf2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e150 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e1c0 blob([0xf4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e230 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e2a0 blob([0xf6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e310 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e380 blob([0xf8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e3f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e460 blob([0xfa000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d0e4d0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194f7a0 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac59d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac4a80 blob([0xe4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9a40 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af9b90 blob([0xe6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9c00 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=25, last_complete=21'7, committed=1, applied=1)) v2 -- 0x558e818a9e00 con 0x558e818fe400 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af9c70 blob([0xe8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9ce0 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af9d50 blob([0xea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9dc0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af9e30 blob([0xec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9ea0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af9f10 blob([0xee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9f80 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0e000 blob([0xf0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e070 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0e0e0 blob([0xf2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e150 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0e1c0 blob([0xf4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e230 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0e2a0 blob([0xf6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e310 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0e380 blob([0xf8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e3f0 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0e460 blob([0xfa000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d0e4d0 sbid 0x0)) need {<0x0, 0x1000> : [0x18000:0~1000]} 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~1000 2019-12-07T19:33:58.801+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194f7a0 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac59d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83ac59e8 in 0x558e80219880) _discard 0x1000~1000 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83d10480 space 0x558e83ac59e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac4a80 blob([0xe4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9a40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af9a58 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83c8f3b0 space 0x558e83af9a58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af9b90 blob([0xe6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9c00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af9c18 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83c8f440 space 0x558e83af9c18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af9c70 blob([0xe8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9ce0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af9cf8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83c8f4d0 space 0x558e83af9cf8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af9d50 blob([0xea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9dc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af9dd8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83c8f560 space 0x558e83af9dd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af9e30 blob([0xec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9ea0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af9eb8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83c8f170 space 0x558e83af9eb8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af9f10 blob([0xee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9f80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af9f98 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83c8fc20 space 0x558e83af9f98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0e000 blob([0xf0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e070 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0e088 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83c8f290 space 0x558e83d0e088 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0e0e0 blob([0xf2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e150 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0e168 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83c8e990 space 0x558e83d0e168 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0e1c0 blob([0xf4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e230 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0e248 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83d11d40 space 0x558e83d0e248 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0e2a0 blob([0xf6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e310 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0e328 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83d115f0 space 0x558e83d0e328 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0e380 blob([0xf8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e3f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0e408 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83d11170 space 0x558e83d0e408 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0e460 blob([0xfa000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d0e4d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18000:0~1000]} 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0e4e8 in 0x558e80219880) _discard 0x0~1000 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83d11200 space 0x558e83d0e4e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.802+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 0x0~19000 = 102400 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read read request=102400 r=102400 len=102400 2019-12-07T19:33:58.802+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: Getting attr on 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.802+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: not in cache 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.802+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.802+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: found on disk, size 102400 2019-12-07T19:33:58.802+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# hinfo_key 2019-12-07T19:33:58.802+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# hinfo_key = 0 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read: Checking hash of 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.802+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 21/18 ECSubReadReply(tid=81, attrs_read=0)) v2 -- 0x558e83c73900 con 0x558e818d7c00 2019-12-07T19:33:58.802+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e83edc840 finish 2019-12-07T19:33:58.802+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.802+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 39 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=27, reqid=client.4540.0:151, at_version=21'8, trim_to=0'0, roll_forward_to=21'6)) v2 ==== 96536+0+0 (unknown 0 0 0) 0x558e83cbf800 con 0x558e818fe400 2019-12-07T19:33:58.802+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0b1e0 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=27, reqid=client.4540.0:151, at_version=21'8, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.802+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=27, reqid=client.4540.0:151, at_version=21'8, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=27, reqid=client.4540.0:151, at_version=21'8, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=27, reqid=client.4540.0:151, at_version=21'8, trim_to=0'0, roll_forward_to=21'6)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e83b0b1e0 prio 127 cost 0 latency 0.000062 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=27, reqid=client.4540.0:151, at_version=21'8, trim_to=0'0, roll_forward_to=21'6)) v2 pg pg[1.bs3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=27, reqid=client.4540.0:151, at_version=21'8, trim_to=0'0, roll_forward_to=21'6)) v2 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'7 (0'0,21'7] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'5 active mbc={}] append_log log((0'0,21'7], crt=21'5) [21'8 (21'6) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:151 2019-12-07T19:33:58.792585+0000 0 ObjectCleanRegions clean_offsets: [0~458752,835584~18446744073708716031], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'7 crt=21'5 lcod 21'7 active mbc={}] add_log_entry 21'8 (21'6) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:151 2019-12-07T19:33:58.792585+0000 0 ObjectCleanRegions clean_offsets: [0~458752,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'7 crt=21'6 lcod 21'7 active mbc={}] rollforward: entry=21'6 (0'0) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:150 2019-12-07T19:33:58.789897+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'7 crt=21'6 lcod 21'7 active mbc={}] append_log approx pg log length = 8 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'7 crt=21'6 lcod 21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.802+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e811fa900 seq 16 2019-12-07T19:33:58.802+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 2 keys 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 2 keys = 0 2019-12-07T19:33:58.802+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:33:58.802+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 0x1c000~17000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:dce63cc3:::smithi13140796-50:head# 0x1c000~17000 - have 0x1c000 (114688) bytes fadvise_flags 0x0 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x1c000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fa900 12 blobs 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate 0x17000/1000,17000,0 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x2f2000~17000/1000,17000,0 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x2f2000~17000] 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84030a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fa40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84030a10 blob([0x2f2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fa40 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e84030a10) get_ref 0x0~2000 Blob(0x558e84030a10 blob([0x2f2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fa40 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e84030a10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e84030a10 blob([0x2f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fa40 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0fa58 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6c310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6c310 blob([0x2f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c380 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6c310) get_ref 0x0~2000 Blob(0x558e83e6c310 blob([0x2f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c380 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6c310) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e83e6c310 blob([0x2f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c380 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6c398 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6c460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6c460 blob([0x2f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c4d0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6c460) get_ref 0x0~2000 Blob(0x558e83e6c460 blob([0x2f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c4d0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6c460) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e83e6c460 blob([0x2f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c4d0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6c4e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6c540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6c540 blob([0x2f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c5b0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6c540) get_ref 0x0~2000 Blob(0x558e83e6c540 blob([0x2f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c5b0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6c540) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e83e6c540 blob([0x2f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c5b0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6c5c8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6c620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6c620 blob([0x2fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c690 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6c620) get_ref 0x0~2000 Blob(0x558e83e6c620 blob([0x2fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c690 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6c620) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e83e6c620 blob([0x2fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c690 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6c6a8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6c700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6df80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6c700 blob([0x2fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6df80 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6c700) get_ref 0x0~2000 Blob(0x558e83e6c700 blob([0x2fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6df80 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6c700) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e83e6c700 blob([0x2fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6df80 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6df98 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6df10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6dea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6df10 blob([0x2fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6dea0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6df10) get_ref 0x0~2000 Blob(0x558e83e6df10 blob([0x2fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6dea0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6df10) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e83e6df10 blob([0x2fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dea0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6deb8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6de30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6ddc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6de30 blob([0x300000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6ddc0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6de30) get_ref 0x0~2000 Blob(0x558e83e6de30 blob([0x300000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6ddc0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6de30) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558e83e6de30 blob([0x300000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6ddc0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6ddd8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6dd50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6dce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6dd50 blob([0x302000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6dce0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6dd50) get_ref 0x0~2000 Blob(0x558e83e6dd50 blob([0x302000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6dce0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6dd50) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558e83e6dd50 blob([0x302000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dce0 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6dcf8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6dc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6dc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6dc70 blob([0x304000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6dc00 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6dc70) get_ref 0x0~2000 Blob(0x558e83e6dc70 blob([0x304000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6dc00 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6dc70) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558e83e6dc70 blob([0x304000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dc00 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6dc18 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6db90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6db20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6db90 blob([0x306000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6db20 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6db90) get_ref 0x0~2000 Blob(0x558e83e6db90 blob([0x306000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6db20 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6db90) get_ref init 0x2000, 1000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558e83e6db90 blob([0x306000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6db20 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6db38 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6dab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6da40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6dab0 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6da40 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6dab0) get_ref 0x0~1000 Blob(0x558e83e6dab0 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6da40 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6dab0) get_ref init 0x1000, 1000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x32000~1000: 0x0~1000 Blob(0x558e83e6dab0 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83e6da40 sbid 0x0)) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6da58 in 0x558e80219c00) _discard 0x0~1000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x33000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83cb2388) dirty_range mark inline shard dirty 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 0x1c000~17000 = 0 2019-12-07T19:33:58.802+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.802+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa900 cost 148186 (13 ios * 4000 + 96186 bytes) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa900 onodes 0x558e83cb2280 shared_blobs 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83cb2388) update 3#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83cb2388) update inline shard 494 bytes from 26 extents 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:dce63cc3:::smithi13140796-50:head# is 913 (413 bytes onode + 2 bytes spanning blobs + 498 bytes inline extents) 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa900 allocated 0x[2f2000~17000] released 0x[] 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 freelist allocate 0x2f2000~17000 2019-12-07T19:33:58.802+0000 7fe2c6263700 20 freelist _xor first_key 0x280000 last_key 0x300000 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 prepare 2019-12-07T19:33:58.802+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e811fa900 2019-12-07T19:33:58.803+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'7 crt=21'6 lcod 21'7 active mbc={}] op_applied version 21'8 2019-12-07T19:33:58.803+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e83b0b1e0 finish 2019-12-07T19:33:58.803+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.803+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 aio_wait 2019-12-07T19:33:58.803+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa900 2019-12-07T19:33:58.803+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 io_done 2019-12-07T19:33:58.803+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.803+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.803+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:58.803+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2082 < min_flush_size 524288 2019-12-07T19:33:58.803+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2280 had 1 2019-12-07T19:33:58.803+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.803+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2105 < min_flush_size 524288 2019-12-07T19:33:58.803+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2105 < min_flush_size 524288 2019-12-07T19:33:58.803+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.803+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x40ea0~839 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.803+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x40ea0 0x40ea0~839 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.803+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.803+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x10ea0 2019-12-07T19:33:58.803+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xea0 2019-12-07T19:33:58.803+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.803+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 40 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=28, reqid=client.4540.0:152, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e83cbe000 con 0x558e818fe400 2019-12-07T19:33:58.803+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0ac60 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=28, reqid=client.4540.0:152, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.803+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=28, reqid=client.4540.0:152, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.803+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 83 ==== MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=82, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8402f400 con 0x558e818d7c00 2019-12-07T19:33:58.803+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83edcb00 prio 127 cost 0 latency 0.000018 epoch 21 MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=82, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.803+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=82, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.803+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 84 ==== MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=83, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e83c73900 con 0x558e818d7c00 2019-12-07T19:33:58.803+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83edcdc0 prio 127 cost 0 latency 0.000015 epoch 21 MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=83, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.803+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=83, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.803+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x6d9 and padding block with 0x927 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=28, reqid=client.4540.0:152, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=28, reqid=client.4540.0:152, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e83b0ac60 prio 127 cost 0 latency 0.000703 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=28, reqid=client.4540.0:152, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.bs3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'6 lcod 21'7 active mbc={}] 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'6 lcod 21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=28, reqid=client.4540.0:152, at_version=21'9, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'6 lcod 21'7 active mbc={}] append_log log((0'0,21'8], crt=21'6) [21'9 (21'8) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:152 2019-12-07T19:33:58.792599+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'8 crt=21'6 lcod 21'7 active mbc={}] add_log_entry 21'9 (21'8) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:152 2019-12-07T19:33:58.792599+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'8 crt=21'7 lcod 21'7 active mbc={}] rollforward: entry=21'7 (21'5) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:147 2019-12-07T19:33:58.786362+0000 0 ObjectCleanRegions clean_offsets: [0~442368,819200~18446744073708732415], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'8 crt=21'7 lcod 21'7 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'8 crt=21'7 lcod 21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.804+0000 7fe2ca26b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.804+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 41 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=29, reqid=client.4540.0:148, at_version=21'10, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e83fea000 con 0x558e818fe400 2019-12-07T19:33:58.804+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0b080 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=29, reqid=client.4540.0:148, at_version=21'10, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.804+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=29, reqid=client.4540.0:148, at_version=21'10, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e83c57500 seq 17 2019-12-07T19:33:58.804+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 3 keys 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=29, reqid=client.4540.0:148, at_version=21'10, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 3 keys = 0 2019-12-07T19:33:58.804+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.804+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c57500 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c57500 onodes 0x558e83cb2280 shared_blobs 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e83cb2388) update 3#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:dce63cc3:::smithi13140796-50:head# is 985 (485 bytes onode + 2 bytes spanning blobs + 498 bytes inline extents) 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c57500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 prepare 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c57500 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 io_done 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'8 crt=21'7 lcod 21'7 active mbc={}] op_applied version 21'9 2019-12-07T19:33:58.804+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e83b0ac60 finish 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x416d9 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=29, reqid=client.4540.0:148, at_version=21'10, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e83b0b080 prio 127 cost 0 latency 0.000121 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=29, reqid=client.4540.0:148, at_version=21'10, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.bs3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.001028284s (0.000043971s flush + 0.000984313s kv commit) 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa900> 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=29, reqid=client.4540.0:148, at_version=21'10, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.804+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 kv_submitted 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa900 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2025 < min_flush_size 524288 2019-12-07T19:33:58.804+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 finishing 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa900 onodes 0x558e83cb2280 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0fa58 in 0x558e80219c00) _finish_write discard buffer(0x558e83d113b0 space 0x558e83d0fa58 0x0~2000 writing nocache) 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2280 had 1 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 lcod 21'7 active mbc={}] append_log log((0'0,21'9], crt=21'7) [21'10 (21'7) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:148 2019-12-07T19:33:58.786376+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6c398 in 0x558e80219c00) _finish_write discard buffer(0x558e84032a20 space 0x558e83e6c398 0x0~2000 writing nocache) 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6c4e8 in 0x558e80219c00) _finish_write discard buffer(0x558e84032990 space 0x558e83e6c4e8 0x0~2000 writing nocache) 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2048 < min_flush_size 524288 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2048 < min_flush_size 524288 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6c5c8 in 0x558e80219c00) _finish_write discard buffer(0x558e84032900 space 0x558e83e6c5c8 0x0~2000 writing nocache) 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6c6a8 in 0x558e80219c00) _finish_write discard buffer(0x558e84032870 space 0x558e83e6c6a8 0x0~2000 writing nocache) 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6da58 in 0x558e80219c00) _finish_write discard buffer(0x558e840323f0 space 0x558e83e6da58 0x0~1000 writing nocache) 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6db38 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8ecf0 space 0x558e83e6db38 0x0~2000 writing nocache) 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6dc18 in 0x558e80219c00) _finish_write discard buffer(0x558e83c8eab0 space 0x558e83e6dc18 0x0~2000 writing nocache) 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x416d9~800 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6dcf8 in 0x558e80219c00) _finish_write discard buffer(0x558e84032000 space 0x558e83e6dcf8 0x0~2000 writing nocache) 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'9 crt=21'7 lcod 21'7 active mbc={}] add_log_entry 21'10 (21'7) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:148 2019-12-07T19:33:58.786376+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6ddd8 in 0x558e80219c00) _finish_write discard buffer(0x558e840326c0 space 0x558e83e6ddd8 0x0~2000 writing nocache) 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6deb8 in 0x558e80219c00) _finish_write discard buffer(0x558e84032750 space 0x558e83e6deb8 0x0~2000 writing nocache) 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x416d9 0x416d9~800 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6df98 in 0x558e80219c00) _finish_write discard buffer(0x558e840327e0 space 0x558e83e6df98 0x0~2000 writing nocache) 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'9 crt=21'7 lcod 21'7 active mbc={}] append_log approx pg log length = 10 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x116d9 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa900 done 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x6d9 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57500 kv_submitted 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.804+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa900 [] 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xed9 and padding block with 0x127 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'9 crt=21'7 lcod 21'7 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.804+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.804+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e811fb500 seq 18 2019-12-07T19:33:58.804+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 3 keys 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 3 keys = 0 2019-12-07T19:33:58.804+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.804+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000010' <- 0000000021.00000000000000000010 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fb500 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=82, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fb500 onodes 0x558e8402ef00 shared_blobs 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 bluestore.extentmap(0x558e8402f008) update 3#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:d313ddd1:::smithi13140796-49:head# is 999 (485 bytes onode + 2 bytes spanning blobs + 512 bytes inline extents) 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=82, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fb500 allocated 0x[] released 0x[] 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 prepare 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fb500 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 io_done 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83edcb00 prio 127 cost 0 latency 0.000637 MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=82, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'9 crt=21'7 lcod 21'7 active mbc={}] op_applied version 21'10 2019-12-07T19:33:58.804+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e83b0b080 finish 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x41ed9 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=82, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000173457s (0.000000158s flush + 0.000173299s kv commit) 2019-12-07T19:33:58.804+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=27, last_complete=21'8, committed=1, applied=1)) v2 -- 0x558e84028200 con 0x558e818fe400 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.804+0000 7fe2c425f700 25 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.804+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 0x0~19000 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~19000 size 0x19000 (102400) 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2039 < min_flush_size 524288 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f7a0 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac59d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ef00 had 1 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c57500> 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4a80 blob([0xe4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9a40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.804+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 kv_submitted 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2062 < min_flush_size 524288 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9b90 blob([0xe6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9c00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2062 < min_flush_size 524288 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c57500 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9c70 blob([0xe8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9ce0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 finishing 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c57500 onodes 0x558e83cb2280 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57500 done 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb500 kv_submitted 2019-12-07T19:33:58.804+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c57500 [] 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x41ed9~80e to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9d50 blob([0xea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9dc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x41ed9 0x41ed9~80e to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9e30 blob([0xec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9ea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9f10 blob([0xee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9f80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x11ed9 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xed9 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x6e7 and padding block with 0x919 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e000 blob([0xf0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e070 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e0e0 blob([0xf2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e150 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=28, last_complete=21'9, committed=1, applied=1)) v2 -- 0x558e818a9e00 con 0x558e818fe400 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e1c0 blob([0xf4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e230 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e2a0 blob([0xf6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e310 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e380 blob([0xf8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e3f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e460 blob([0xfa000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d0e4d0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 0x0~19000 = 102400 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read read request=102400 r=102400 len=102400 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: Getting attr on 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: not in cache 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: found on disk, size 102400 2019-12-07T19:33:58.804+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# hinfo_key 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# hinfo_key = 0 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read: Checking hash of 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x426e7 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.804+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.804+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 21/18 ECSubReadReply(tid=82, attrs_read=0)) v2 -- 0x558e83c73680 con 0x558e818d7c00 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000133855s (0.000000095s flush + 0.000133760s kv commit) 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83edcb00 finish 2019-12-07T19:33:58.804+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fb500> 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.804+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 kv_submitted 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fb500 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.804+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 finishing 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fb500 onodes 0x558e8402ef00 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb500 done 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=83, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.804+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fb500 [] 2019-12-07T19:33:58.804+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.804+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=83, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.804+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.804+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=29, last_complete=21'10, committed=1, applied=1)) v2 -- 0x558e84029e00 con 0x558e818fe400 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83edcdc0 prio 127 cost 0 latency 0.000801 MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=83, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.804+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpRead(1.7s3 21/18 ECSubRead(tid=83, to_read={1:ee170468:::smithi13140796-25:head=0,102400,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.804+0000 7fe2c425f700 25 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.804+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 0x0~19000 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~19000 size 0x19000 (102400) 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f7a0 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac59d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4a80 blob([0xe4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9a40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9b90 blob([0xe6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9c00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9c70 blob([0xe8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9ce0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9d50 blob([0xea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9dc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9e30 blob([0xec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9ea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9f10 blob([0xee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9f80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e000 blob([0xf0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e070 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e0e0 blob([0xf2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e150 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e1c0 blob([0xf4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e230 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e2a0 blob([0xf6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e310 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e380 blob([0xf8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e3f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e460 blob([0xfa000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d0e4d0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 0x0~19000 = 102400 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read read request=102400 r=102400 len=102400 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: Getting attr on 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: not in cache 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] get_hash_info: found on disk, size 102400 2019-12-07T19:33:58.804+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# hinfo_key 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# hinfo_key = 0 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] handle_sub_read: Checking hash of 1:ee170468:::smithi13140796-25:head 2019-12-07T19:33:58.804+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 21/18 ECSubReadReply(tid=83, attrs_read=0)) v2 -- 0x558e8402f400 con 0x558e818d7c00 2019-12-07T19:33:58.804+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83edcdc0 finish 2019-12-07T19:33:58.804+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.805+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 42 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=30, reqid=client.4540.0:158, at_version=21'11, trim_to=0'0, roll_forward_to=21'7)) v2 ==== 33092+0+0 (unknown 0 0 0) 0x558e83fed000 con 0x558e818fe400 2019-12-07T19:33:58.805+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0b340 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=30, reqid=client.4540.0:158, at_version=21'11, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.805+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=30, reqid=client.4540.0:158, at_version=21'11, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=30, reqid=client.4540.0:158, at_version=21'11, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=30, reqid=client.4540.0:158, at_version=21'11, trim_to=0'0, roll_forward_to=21'7)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e83b0b340 prio 127 cost 0 latency 0.000075 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=30, reqid=client.4540.0:158, at_version=21'11, trim_to=0'0, roll_forward_to=21'7)) v2 pg pg[1.bs3( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=30, reqid=client.4540.0:158, at_version=21'11, trim_to=0'0, roll_forward_to=21'7)) v2 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'10 (0'0,21'10] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'10], crt=21'7) [21'11 (21'4) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:158 2019-12-07T19:33:58.802344+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'10 crt=21'7 lcod 21'10 active mbc={}] add_log_entry 21'11 (21'4) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:158 2019-12-07T19:33:58.802344+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'10 crt=21'7 lcod 21'10 active mbc={}] append_log approx pg log length = 11 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'10 crt=21'7 lcod 21'10 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.805+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e811fa600 seq 19 2019-12-07T19:33:58.805+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# 22 keys 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# 22 keys = 0 2019-12-07T19:33:58.805+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.805+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000011' <- 0000000021.00000000000000000011 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa600 cost 36742 (1 ios * 4000 + 32742 bytes) 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa600 onodes 0x558e83cb2000 shared_blobs 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83cb2108) update 3#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:d07269f7:::smithi13140796-28:head# is 30805 (30530 bytes onode + 2 bytes spanning blobs + 273 bytes inline extents) 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa600 allocated 0x[] released 0x[] 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 prepare 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa600 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 io_done 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'10 crt=21'7 lcod 21'10 active mbc={}] op_applied version 21'11 2019-12-07T19:33:58.805+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e83b0b340 finish 2019-12-07T19:33:58.805+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.805+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.805+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.805+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 22809 < min_flush_size 524288 2019-12-07T19:33:58.805+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 32742 < min_flush_size 524288 2019-12-07T19:33:58.805+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2000 had 1 2019-12-07T19:33:58.805+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.805+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 32765 < min_flush_size 524288 2019-12-07T19:33:58.805+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 32765 < min_flush_size 524288 2019-12-07T19:33:58.805+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.805+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x426e7~7ffd to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.805+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x426e7 0x426e7~7ffd to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.805+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.805+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x126e7 2019-12-07T19:33:58.805+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x6e7 2019-12-07T19:33:58.805+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.805+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x6e4 and padding block with 0x91c 2019-12-07T19:33:58.805+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x4a6e4 2019-12-07T19:33:58.805+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.805+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.805+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.805+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.806+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000205645s (0.000000067s flush + 0.000205578s kv commit) 2019-12-07T19:33:58.806+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.806+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.806+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa600> 2019-12-07T19:33:58.806+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.806+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 kv_submitted 2019-12-07T19:33:58.806+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa600 2019-12-07T19:33:58.806+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 finishing 2019-12-07T19:33:58.806+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa600 onodes 0x558e83cb2000 2019-12-07T19:33:58.806+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa600 done 2019-12-07T19:33:58.806+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:58.806+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa600 [] 2019-12-07T19:33:58.806+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.806+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.806+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=30, last_complete=21'11, committed=1, applied=1)) v2 -- 0x558e84028000 con 0x558e818fe400 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.806+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.806+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 43 ==== MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=31, to_read={1:dce63cc3:::smithi13140796-50:head=0,4096,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8402ec80 con 0x558e818fe400 2019-12-07T19:33:58.806+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a2c0 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=31, to_read={1:dce63cc3:::smithi13140796-50:head=0,4096,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.806+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=31, to_read={1:dce63cc3:::smithi13140796-50:head=0,4096,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=31, to_read={1:dce63cc3:::smithi13140796-50:head=0,4096,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=31, to_read={1:dce63cc3:::smithi13140796-50:head=0,4096,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.806+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e83b0a2c0 prio 127 cost 0 latency 0.000052 MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=31, to_read={1:dce63cc3:::smithi13140796-50:head=0,4096,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.806+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=31, to_read={1:dce63cc3:::smithi13140796-50:head=0,4096,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.806+0000 7fe2ca26b700 25 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.806+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 0x0~1000 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x33000 (208896) 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d02620 blob([0x2bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48e00 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d02620 blob([0x2bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48e00 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d02620 blob([0x2bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48e00 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83f48e18 in 0x558e80219c00) _discard 0x0~1000 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e84032360 space 0x558e83f48e18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.806+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 0x0~1000 = 4096 2019-12-07T19:33:58.806+0000 7fe2ca26b700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.806+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] get_hash_info: Getting attr on 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:33:58.807+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] get_hash_info: not in cache 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:33:58.807+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:33:58.807+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] get_hash_info: found on disk, size 208896 2019-12-07T19:33:58.807+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# hinfo_key 2019-12-07T19:33:58.807+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# hinfo_key = 0 2019-12-07T19:33:58.807+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 21/18 ECSubReadReply(tid=31, attrs_read=0)) v2 -- 0x558e8402e500 con 0x558e818fe400 2019-12-07T19:33:58.807+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e83b0a2c0 finish 2019-12-07T19:33:58.807+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.810+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 44 ==== MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=32, to_read={1:d313ddd1:::smithi13140796-49:head=0,4096,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8402e500 con 0x558e818fe400 2019-12-07T19:33:58.810+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a9a0 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=32, to_read={1:d313ddd1:::smithi13140796-49:head=0,4096,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.810+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=32, to_read={1:d313ddd1:::smithi13140796-49:head=0,4096,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=32, to_read={1:d313ddd1:::smithi13140796-49:head=0,4096,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=32, to_read={1:d313ddd1:::smithi13140796-49:head=0,4096,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.810+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 45 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=33, reqid=client.4540.0:158, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83aca800 con 0x558e818fe400 2019-12-07T19:33:58.810+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e83b0a9a0 prio 127 cost 0 latency 0.000094 MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=32, to_read={1:d313ddd1:::smithi13140796-49:head=0,4096,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.810+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0a000 prio 127 cost 0 latency 0.000063 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=33, reqid=client.4540.0:158, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.810+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=33, reqid=client.4540.0:158, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.810+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpRead(1.bs3 21/18 ECSubRead(tid=32, to_read={1:d313ddd1:::smithi13140796-49:head=0,4096,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=33, reqid=client.4540.0:158, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.810+0000 7fe2c6263700 25 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.810+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~1000 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x32000 (204800) 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e700 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9ce0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194e700 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9ce0 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194e700 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9ce0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e817e9cf8 in 0x558e80219c00) _discard 0x0~1000 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e83c8ebd0 space 0x558e817e9cf8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.810+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~1000 = 4096 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:33:58.810+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] get_hash_info: Getting attr on 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:33:58.810+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] get_hash_info: not in cache 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:33:58.810+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:33:58.810+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] get_hash_info: found on disk, size 204800 2019-12-07T19:33:58.810+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key 2019-12-07T19:33:58.810+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key = 0 2019-12-07T19:33:58.810+0000 7fe2c6263700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 21/18 ECSubReadReply(tid=32, attrs_read=0)) v2 -- 0x558e83c73180 con 0x558e818fe400 2019-12-07T19:33:58.810+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e83b0a9a0 finish 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=33, reqid=client.4540.0:158, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.810+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e83b0a000 prio 127 cost 0 latency 0.000408 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=33, reqid=client.4540.0:158, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 pg pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] 2019-12-07T19:33:58.810+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=33, reqid=client.4540.0:158, at_version=0'0, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.810+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'7 active mbc={}] append_log log((0'0,21'11], crt=21'7) [] 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'11 active mbc={}] rollforward: entry=21'8 (21'6) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:151 2019-12-07T19:33:58.792585+0000 0 ObjectCleanRegions clean_offsets: [0~458752,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'11 active mbc={}] rollforward: entry=21'9 (21'8) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:152 2019-12-07T19:33:58.792599+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'11 active mbc={}] rollforward: entry=21'10 (21'7) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:148 2019-12-07T19:33:58.786376+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'11 active mbc={}] rollforward: entry=21'11 (21'4) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:158 2019-12-07T19:33:58.802344+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.810+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log approx pg log length = 11 2019-12-07T19:33:58.810+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.810+0000 7fe2ca26b700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.810+0000 7fe2ca26b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.810+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e83c56300 seq 20 2019-12-07T19:33:58.810+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.810+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.810+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56300 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56300 onodes shared_blobs 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.810+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 prepare 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56300 2019-12-07T19:33:58.810+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 io_done 2019-12-07T19:33:58.810+0000 7fe2ca26b700 10 osd.0 21 dequeue_op 0x558e83b0a000 finish 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.810+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.810+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.810+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.810+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.810+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.810+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.810+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.810+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x4a6e4~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.810+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x4a6e4 0x4a6e4~471 to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.810+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.810+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1a6e4 2019-12-07T19:33:58.810+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x6e4 2019-12-07T19:33:58.810+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.810+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb55 and padding block with 0x4ab 2019-12-07T19:33:58.810+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x4ab55 2019-12-07T19:33:58.810+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.810+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.810+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.810+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.810+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000093495s (0.000000110s flush + 0.000093385s kv commit) 2019-12-07T19:33:58.810+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.810+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.810+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56300> 2019-12-07T19:33:58.810+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.810+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 kv_submitted 2019-12-07T19:33:58.810+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56300 2019-12-07T19:33:58.810+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 finishing 2019-12-07T19:33:58.810+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56300 onodes 2019-12-07T19:33:58.810+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56300 done 2019-12-07T19:33:58.810+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:58.810+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56300 [] 2019-12-07T19:33:58.810+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.810+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.810+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.810+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=33, last_complete=21'11, committed=1, applied=1)) v2 -- 0x558e84029e00 con 0x558e818fe400 2019-12-07T19:33:58.810+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.819+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 85 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 ==== 1364+0+0 (unknown 0 0 0) 0x558e83e56000 con 0x558e818d7c00 2019-12-07T19:33:58.819+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83edcf20 prio 127 cost 0 latency 0.000029 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.819+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e83edcf20 prio 127 cost 0 latency 0.000063 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 pg pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=84, reqid=client.4540.0:164, at_version=21'9, trim_to=0'0, roll_forward_to=21'8)) v2 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'8 (0'0,21'8] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log log((0'0,21'8], crt=21'8) [21'9 (21'8) delete 1:e7f24557:::smithi13140796-34:head by client.4540.0:164 2019-12-07T19:33:58.819645+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 0, new_object: 0] 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] add_log_entry 21'9 (21'8) delete 1:e7f24557:::smithi13140796-34:head by client.4540.0:164 2019-12-07T19:33:58.819645+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.819+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e81a0a300 seq 20 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.7s3_head 0x558e8122c5a0) get_onode oid 3#1:e7f24557:::smithi13140796-34:head#9 key 0x838000000000000001e7f245'W!smithi13140796-34!='0xfffffffffffffffe0000000000000009'o' 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.7s3_head 0x558e8122c5a0) r -2 v.len 0 2019-12-07T19:33:58.819+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# -> 3#1:e7f24557:::smithi13140796-34:head#9 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head# -> 3#1:e7f24557:::smithi13140796-34:head#9 = 0 2019-12-07T19:33:58.819+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a300 cost 5014 (1 ios * 4000 + 1014 bytes) 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a300 onodes 0x558e83b09180 shared_blobs 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83b09288) update 3#1:e7f24557:::smithi13140796-34:head#9 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:e7f24557:::smithi13140796-34:head#9 is 817 (485 bytes onode + 2 bytes spanning blobs + 330 bytes inline extents) 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a300 allocated 0x[] released 0x[] 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 prepare 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a300 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 io_done 2019-12-07T19:33:58.819+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.819+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 lua=21'8 crt=21'8 lcod 21'8 active mbc={}] op_applied version 21'9 2019-12-07T19:33:58.819+0000 7fe2c8267700 10 osd.0 21 dequeue_op 0x558e83edcf20 finish 2019-12-07T19:33:58.819+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1509 < min_flush_size 524288 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.819+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b09180 had 1 2019-12-07T19:33:58.819+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6000 had 1 2019-12-07T19:33:58.819+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.819+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1532 < min_flush_size 524288 2019-12-07T19:33:58.819+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1532 < min_flush_size 524288 2019-12-07T19:33:58.819+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.819+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x4ab55~5fc to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.819+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x4ab55 0x4ab55~5fc to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.819+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.819+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~20000 x_off 0x1ab55 2019-12-07T19:33:58.819+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb55 2019-12-07T19:33:58.819+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.819+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x151 and padding block with 0xeaf 2019-12-07T19:33:58.819+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x4b151 2019-12-07T19:33:58.819+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.819+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.819+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.819+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.819+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000103557s (0.000000058s flush + 0.000103499s kv commit) 2019-12-07T19:33:58.819+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.819+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0a300> 2019-12-07T19:33:58.819+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.819+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.819+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 kv_submitted 2019-12-07T19:33:58.819+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a300 2019-12-07T19:33:58.819+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 finishing 2019-12-07T19:33:58.819+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a300 onodes 0x558e83b09180 2019-12-07T19:33:58.819+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a300 done 2019-12-07T19:33:58.819+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.819+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a300 [] 2019-12-07T19:33:58.819+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.819+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.819+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.819+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=84, last_complete=21'9, committed=1, applied=1)) v2 -- 0x558e83fefe00 con 0x558e818d7c00 2019-12-07T19:33:58.819+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.820+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 45 ==== MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=45, reqid=client.4540.0:163, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 56941+0+0 (unknown 0 0 0) 0x558e83e57800 con 0x558e818d7000 2019-12-07T19:33:58.820+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83edd080 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=45, reqid=client.4540.0:163, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.820+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=45, reqid=client.4540.0:163, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=45, reqid=client.4540.0:163, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=45, reqid=client.4540.0:163, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e21) pg 0x558e819e4000 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e83edd080 prio 127 cost 0 latency 0.000128 MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=45, reqid=client.4540.0:163, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=45, reqid=client.4540.0:163, at_version=21'3, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'2], crt=21'2) [21'3 (21'2) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:163 2019-12-07T19:33:58.819566+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] add_log_entry 21'3 (21'2) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:163 2019-12-07T19:33:58.819566+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.820+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e83c56000 seq 9 2019-12-07T19:33:58.820+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 22 keys 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 22 keys = 0 2019-12-07T19:33:58.820+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) = 0 2019-12-07T19:33:58.820+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56000 cost 60591 (1 ios * 4000 + 56591 bytes) 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56000 onodes 0x558e811dd680 shared_blobs 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dd788) update 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:891f0d13:::smithi13140796-14:head# is 54532 (54390 bytes onode + 2 bytes spanning blobs + 140 bytes inline extents) 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56000 allocated 0x[] released 0x[] 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 prepare 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56000 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 io_done 2019-12-07T19:33:58.820+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.820+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={}] op_applied version 21'3 2019-12-07T19:33:58.820+0000 7fe2c7265700 10 osd.0 21 dequeue_op 0x558e83edd080 finish 2019-12-07T19:33:58.820+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.820+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 20143 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 52911 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 56469 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dd680 had 1 2019-12-07T19:33:58.820+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:33:58.820+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 56492 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 56492 < min_flush_size 524288 2019-12-07T19:33:58.820+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.820+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x4b151~dcac to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.820+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x4b151 0x4b151~dcac to file(ino 16 size 0x50000 mtime 2019-12-07T19:33:58.796380+0000 allocated 50000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~20000]) 2019-12-07T19:33:58.820+0000 7fe2d7a86700 10 bluefs _allocate len 0x8dfd from 0 2019-12-07T19:33:58.820+0000 7fe2d7a86700 10 bluefs _allocate len 0x8dfd from 1 2019-12-07T19:33:58.820+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd180000 2019-12-07T19:33:58.820+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd180000~10000/10000,10000,acd180000 2019-12-07T19:33:58.820+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x60000 to include allocated 2019-12-07T19:33:58.820+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 27 (was clean) 2019-12-07T19:33:58.820+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.821156+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.820+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 86 ==== MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83e55800 con 0x558e818d7c00 2019-12-07T19:33:58.820+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83edc9a0 prio 127 cost 0 latency 0.000028 epoch 21 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.820+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2) prio 127 cost 0 e21) pg 0x558e8122e000 2019-12-07T19:33:58.820+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83edc9a0 prio 127 cost 0 latency 0.000075 MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 pg pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] 2019-12-07T19:33:58.820+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] _handle_message: MOSDECSubOpWrite(1.7s3 21/18 ECSubWrite(tid=85, reqid=client.4540.0:164, at_version=0'0, trim_to=0'0, roll_forward_to=21'9)) v2 2019-12-07T19:33:58.820+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'8 active mbc={}] append_log log((0'0,21'9], crt=21'8) [] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 osd.0 pg_epoch: 21 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] rollforward: entry=21'9 (21'8) delete 1:e7f24557:::smithi13140796-34:head by client.4540.0:164 2019-12-07T19:33:58.819645+0000 0 ObjectCleanRegions clean_offsets: [557056~18446744073708994559], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.820+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log approx pg log length = 9 2019-12-07T19:33:58.820+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.820+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.820+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.820+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e8195f500 seq 21 2019-12-07T19:33:58.820+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head#9 onode 0x558e83b09180 txc 0x558e8195f500 2019-12-07T19:33:58.820+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head#9 0x0 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83c65030) put_ref 0x0~2000 Blob(0x558e83c65030 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5f10 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37ea0) put_ref 0x0~2000 Blob(0x558e83d37ea0 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37f10 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37f80) put_ref 0x0~2000 Blob(0x558e83d37f80 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03180 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d031f0) put_ref 0x0~2000 Blob(0x558e83d031f0 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03260 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d032d0) put_ref 0x0~2000 Blob(0x558e83d032d0 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03340 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d033b0) put_ref 0x0~2000 Blob(0x558e83d033b0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03420 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03490) put_ref 0x0~2000 Blob(0x558e83d03490 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03570) put_ref 0x0~2000 Blob(0x558e83d03570 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d035e0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03650) put_ref 0x0~2000 Blob(0x558e83d03650 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d036c0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03730) put_ref 0x0~2000 Blob(0x558e83d03730 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d037a0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03810) put_ref 0x0~2000 Blob(0x558e83d03810 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03880 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d038f0) put_ref 0x0~2000 Blob(0x558e83d038f0 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03960 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d039d0) put_ref 0x0~2000 Blob(0x558e83d039d0 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03a40 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03ab0) put_ref 0x0~2000 Blob(0x558e83d03ab0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03b20 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03b90) put_ref 0x0~2000 Blob(0x558e83d03b90 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03c00 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03c70) put_ref 0x0~2000 Blob(0x558e83d03c70 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03ce0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03d50) put_ref 0x0~2000 Blob(0x558e83d03d50 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03dc0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83b09288) dirty_range mark inline shard dirty 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e83c65030 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83ac5f10 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x156000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83c65030) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5f28 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 _rm buffer(0x558e81170c60 space 0x558e83ac5f28 0x0~1000 clean) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x156000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83ac5f10) put 0x558e83ac5f10 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5f28 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e83d37ea0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d37f10 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x158000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37ea0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d37f28 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x158000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d37f10) put 0x558e83d37f10 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d37f28 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e83d37f80 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d03180 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x15a000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d37f80) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03198 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x15a000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d03180) put 0x558e83d03180 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03198 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e83d031f0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d03260 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x15c000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d031f0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03278 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x15c000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d03260) put 0x558e83d03260 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03278 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e83d032d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d03340 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x15e000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d032d0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03358 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x15e000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d03340) put 0x558e83d03340 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03358 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e83d033b0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d03420 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x160000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d033b0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03438 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x160000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d03420) put 0x558e83d03420 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03438 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e83d03490 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d03500 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x162000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03490) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03518 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x162000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d03500) put 0x558e83d03500 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03518 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558e83d03570 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d035e0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x164000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03570) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d035f8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x164000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d035e0) put 0x558e83d035e0 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d035f8 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558e83d03650 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d036c0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x166000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03650) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d036d8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x166000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d036c0) put 0x558e83d036c0 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d036d8 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558e83d03730 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d037a0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x168000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03730) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d037b8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x168000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d037a0) put 0x558e83d037a0 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d037b8 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558e83d03810 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d03880 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x16a000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03810) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03898 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x16a000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d03880) put 0x558e83d03880 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03898 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558e83d038f0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d03960 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x16c000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d038f0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03978 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x16c000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d03960) put 0x558e83d03960 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03978 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558e83d039d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d03a40 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x16e000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d039d0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03a58 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x16e000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d03a40) put 0x558e83d03a40 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03a58 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558e83d03ab0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d03b20 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x170000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03ab0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03b38 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x170000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d03b20) put 0x558e83d03b20 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03b38 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558e83d03b90 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d03c00 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x172000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03b90) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03c18 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x172000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d03c00) put 0x558e83d03c00 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03c18 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558e83d03c70 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d03ce0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x174000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03c70) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03cf8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x174000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d03ce0) put 0x558e83d03ce0 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03cf8 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558e83d03d50 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d03dc0 sbid 0x0)) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x176000~2000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.blob(0x558e83d03d50) discard_unallocated 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03dd8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x176000~2000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.sharedblob(0x558e83d03dc0) put 0x558e83d03dc0 removing self from set 0x558e8122c670 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03dd8 in 0x558e80219880) _clear 2019-12-07T19:33:58.820+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.7s3_head 3#1:e7f24557:::smithi13140796-34:head#9 = 0 2019-12-07T19:33:58.820+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.820+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:33:58.820+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195f500 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195f500 onodes shared_blobs 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195f500 allocated 0x[] released 0x[156000~22000] 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv release 0x156000~22000 2019-12-07T19:33:58.820+0000 7fe2c425f700 10 freelist release 0x156000~22000 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:33:58.820+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 prepare 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195f500 2019-12-07T19:33:58.820+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 io_done 2019-12-07T19:33:58.820+0000 7fe2c425f700 10 osd.0 21 dequeue_op 0x558e83edc9a0 finish 2019-12-07T19:33:58.820+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.820+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~30000 x_off 0x1b151 2019-12-07T19:33:58.820+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x151 2019-12-07T19:33:58.820+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.820+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xdfd and padding block with 0x203 2019-12-07T19:33:58.820+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x60000 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x58dfd 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.821+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 46 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=34, reqid=client.4540.0:165, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 ==== 1364+0+0 (unknown 0 0 0) 0x558e83ac9000 con 0x558e818fe400 2019-12-07T19:33:58.821+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0ba20 prio 127 cost 0 latency 0.000027 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=34, reqid=client.4540.0:165, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.821+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=34, reqid=client.4540.0:165, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (27) on file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.821156+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]), flushing log 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.821156+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 27 len 0x49 crc 0x498c332d) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x1a000~1000 to file(ino 1 size 0x1a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x1a000 0x1a000~1000 to file(ino 1 size 0x1a000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x1b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1a000 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=34, reqid=client.4540.0:165, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=34, reqid=client.4540.0:165, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x1b000 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e83b0ba20 prio 127 cost 0 latency 0.000113 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=34, reqid=client.4540.0:165, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 pg pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'11 active mbc={}] 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'11 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=34, reqid=client.4540.0:165, at_version=21'12, trim_to=0'0, roll_forward_to=21'11)) v2 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'11 (0'0,21'11] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log log((0'0,21'11], crt=21'11) [21'12 (21'11) delete 1:d07269f7:::smithi13140796-28:head by client.4540.0:165 2019-12-07T19:33:58.821029+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 0, new_object: 0] 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] add_log_entry 21'12 (21'11) delete 1:d07269f7:::smithi13140796-28:head by client.4540.0:165 2019-12-07T19:33:58.821029+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] append_log approx pg log length = 12 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.821+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 21'12, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e811fac00 seq 21 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs3_head 0x558e819c41e0) get_onode oid 3#1:d07269f7:::smithi13140796-28:head#c key 0x838000000000000001d07269f721736d'ithi13140796-28!='0xfffffffffffffffe000000000000000c'o' 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs3_head 0x558e819c41e0) r -2 v.len 0 2019-12-07T19:33:58.821+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# -> 3#1:d07269f7:::smithi13140796-28:head#c 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head# -> 3#1:d07269f7:::smithi13140796-28:head#c = 0 2019-12-07T19:33:58.821+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.0000000021.00000000000000000012' <- 0000000021.00000000000000000012 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._fastinfo' <- _fastinfo 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fac00 cost 5014 (1 ios * 4000 + 1014 bytes) 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fac00 onodes 0x558e83cb2000 shared_blobs 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83cb2108) update 3#1:d07269f7:::smithi13140796-28:head#c 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:d07269f7:::smithi13140796-28:head#c is 30805 (30530 bytes onode + 2 bytes spanning blobs + 273 bytes inline extents) 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fac00 allocated 0x[] released 0x[] 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000118 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 prepare 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fac00 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 io_done 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 lua=21'11 crt=21'11 lcod 21'11 active mbc={}] op_applied version 21'12 2019-12-07T19:33:58.821+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e83b0ba20 finish 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 27 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.821156+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.001011352s (0.000000074s flush + 0.001011278s kv commit) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1250 < min_flush_size 524288 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56000> 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.821+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 kv_submitted 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56000 2019-12-07T19:33:58.821+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 finishing 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56000 onodes 0x558e811dd680 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56000 done 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b09180 had 1 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:33:58.821+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56000 [] 2019-12-07T19:33:58.821+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 29187 < min_flush_size 524288 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 32759 < min_flush_size 524288 2019-12-07T19:33:58.821+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 21/18 ECSubWriteReply(tid=45, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e81955c00 con 0x558e818d7000 2019-12-07T19:33:58.821+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2000 had 1 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2500 had 1 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 32782 < min_flush_size 524288 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 32782 < min_flush_size 524288 2019-12-07T19:33:58.821+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.821156+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x58dfd~800e to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.821156+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x58dfd 0x58dfd~800e to file(ino 16 size 0x60000 mtime 2019-12-07T19:33:58.821156+0000 allocated 60000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~30000]) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _allocate len 0xe0b from 0 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _allocate len 0xe0b from 1 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd190000 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd190000~10000/10000,10000,acd190000 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x70000 to include allocated 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 28 (was clean) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x28dfd 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xdfd 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xe0b and padding block with 0x1f5 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x70000 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x60e0b 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (28) on file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]), flushing log 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 28 len 0x49 crc 0xd2c5e2c8) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x1b000~1000 to file(ino 1 size 0x1b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x1b000 0x1b000~1000 to file(ino 1 size 0x1b000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x1c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1b000 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x1c000 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:33:58.821+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000017 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 28 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000439361s (0.000000078s flush + 0.000439283s kv commit) 2019-12-07T19:33:58.821+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195f500,0x558e811fac00> 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.821+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 kv_submitted 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195f500 2019-12-07T19:33:58.821+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 finishing 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195f500 onodes 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f500 done 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:33:58.821+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195f500 [156000~22000] 2019-12-07T19:33:58.821+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release 0x156000~22000 2019-12-07T19:33:58.821+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.821+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 kv_submitted 2019-12-07T19:33:58.821+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fac00 2019-12-07T19:33:58.821+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 finishing 2019-12-07T19:33:58.821+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.7s0 21/18 ECSubWriteReply(tid=85, last_complete=21'9, committed=1, applied=1)) v2 -- 0x558e811f6600 con 0x558e818d7c00 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fac00 onodes 0x558e83cb2000 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fac00 done 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:58.821+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fac00 [] 2019-12-07T19:33:58.821+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.821+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.821+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.821+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:58.821+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=34, last_complete=21'12, committed=1, applied=1)) v2 -- 0x558e83fef800 con 0x558e818fe400 2019-12-07T19:33:58.821+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.822+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 47 ==== MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=35, reqid=client.4540.0:165, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83fec000 con 0x558e818fe400 2019-12-07T19:33:58.822+0000 7fe2e6d3a700 15 osd.0 21 enqueue_op 0x558e83b0b1e0 prio 127 cost 0 latency 0.000020 epoch 21 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=35, reqid=client.4540.0:165, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:33:58.822+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=35, reqid=client.4540.0:165, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=35, reqid=client.4540.0:165, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=35, reqid=client.4540.0:165, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2) prio 127 cost 0 e21) pg 0x558e81928000 2019-12-07T19:33:58.822+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e83b0b1e0 prio 127 cost 0 latency 0.000054 MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=35, reqid=client.4540.0:165, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 pg pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'11 active mbc={}] 2019-12-07T19:33:58.822+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'11 active mbc={}] _handle_message: MOSDECSubOpWrite(1.bs3 21/18 ECSubWrite(tid=35, reqid=client.4540.0:165, at_version=0'0, trim_to=0'0, roll_forward_to=21'12)) v2 2019-12-07T19:33:58.822+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'11 active mbc={}] append_log log((0'0,21'12], crt=21'11) [] 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 osd.0 pg_epoch: 21 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] rollforward: entry=21'12 (21'11) delete 1:d07269f7:::smithi13140796-28:head by client.4540.0:165 2019-12-07T19:33:58.821029+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 0, new_object: 0 2019-12-07T19:33:58.822+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] append_log approx pg log length = 12 2019-12-07T19:33:58.822+0000 7fe2c6263700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.822+0000 7fe2c6263700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.822+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.822+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e8195fb00 seq 22 2019-12-07T19:33:58.822+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head#c onode 0x558e83cb2000 txc 0x558e8195fb00 2019-12-07T19:33:58.822+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head#c 0x0 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d36bd0) put_ref 0x0~2000 Blob(0x558e83d36bd0 blob([0xfb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f880 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d36ee0) put_ref 0x0~2000 Blob(0x558e83d36ee0 blob([0xfd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36f50 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d370a0) put_ref 0x0~2000 Blob(0x558e83d370a0 blob([0xff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37110 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d37180) put_ref 0x0~2000 Blob(0x558e83d37180 blob([0x101000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d371f0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d37260) put_ref 0x0~2000 Blob(0x558e83d37260 blob([0x103000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d372d0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0ef50) put_ref 0x0~2000 Blob(0x558e83d0ef50 blob([0x105000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0efc0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f030) put_ref 0x0~2000 Blob(0x558e83d0f030 blob([0x107000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f0a0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f110) put_ref 0x0~2000 Blob(0x558e83d0f110 blob([0x109000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e5b0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e8194fa40) put_ref 0x0~2000 Blob(0x558e8194fa40 blob([0x10b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81866bd0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d373b0) put_ref 0x0~2000 Blob(0x558e83d373b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37420 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d37340) put_ref 0x0~2000 Blob(0x558e83d37340 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9490 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f180) put_ref 0x0~2000 Blob(0x558e83d0f180 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f1f0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f260) put_ref 0x0~2000 Blob(0x558e83d0f260 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f340) put_ref 0x0~2000 Blob(0x558e83d0f340 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f3b0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83cb2108) dirty_range mark inline shard dirty 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e83d36bd0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8194f880 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xfb000~2000] 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d36bd0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e8194f898 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 _rm buffer(0x558e83c8e6c0 space 0x558e8194f898 0x0~1000 clean) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0xfb000~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.sharedblob(0x558e8194f880) put 0x558e8194f880 removing self from set 0x558e819c42b0 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e8194f898 in 0x558e80219c00) _clear 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e83d36ee0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d36f50 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xfd000~2000] 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d36ee0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d36f68 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0xfd000~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.sharedblob(0x558e83d36f50) put 0x558e83d36f50 removing self from set 0x558e819c42b0 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d36f68 in 0x558e80219c00) _clear 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e83d370a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d37110 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xff000~2000] 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d370a0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d37128 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0xff000~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.sharedblob(0x558e83d37110) put 0x558e83d37110 removing self from set 0x558e819c42b0 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d37128 in 0x558e80219c00) _clear 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e83d37180 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d371f0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x101000~2000] 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d37180) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d37208 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x101000~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.sharedblob(0x558e83d371f0) put 0x558e83d371f0 removing self from set 0x558e819c42b0 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d37208 in 0x558e80219c00) _clear 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e83d37260 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d372d0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x103000~2000] 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d37260) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d372e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x103000~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.sharedblob(0x558e83d372d0) put 0x558e83d372d0 removing self from set 0x558e819c42b0 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d372e8 in 0x558e80219c00) _clear 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e83d0ef50 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d0efc0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x105000~2000] 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0ef50) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0efd8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x105000~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.sharedblob(0x558e83d0efc0) put 0x558e83d0efc0 removing self from set 0x558e819c42b0 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0efd8 in 0x558e80219c00) _clear 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e83d0f030 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d0f0a0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x107000~2000] 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f030) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0f0b8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x107000~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.sharedblob(0x558e83d0f0a0) put 0x558e83d0f0a0 removing self from set 0x558e819c42b0 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0f0b8 in 0x558e80219c00) _clear 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558e83d0f110 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8194e5b0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x109000~2000] 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f110) discard_unallocated 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e8194e5c8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x109000~2000 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.sharedblob(0x558e8194e5b0) put 0x558e8194e5b0 removing self from set 0x558e819c42b0 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e8194e5c8 in 0x558e80219c00) _clear 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558e8194fa40 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e81866bd0 sbid 0x0)) 2019-12-07T19:33:58.822+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x10b000~2000] 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.blob(0x558e8194fa40) discard_unallocated 0x0~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81866be8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x10b000~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.sharedblob(0x558e81866bd0) put 0x558e81866bd0 removing self from set 0x558e819c42b0 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81866be8 in 0x558e80219c00) _clear 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558e83d373b0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d37420 sbid 0x0)) 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x10d000~2000] 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.blob(0x558e83d373b0) discard_unallocated 0x0~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d37438 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x10d000~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.sharedblob(0x558e83d37420) put 0x558e83d37420 removing self from set 0x558e819c42b0 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d37438 in 0x558e80219c00) _clear 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558e83d37340 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e817e9490 sbid 0x0)) 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x10f000~2000] 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.blob(0x558e83d37340) discard_unallocated 0x0~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e817e94a8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x10f000~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.sharedblob(0x558e817e9490) put 0x558e817e9490 removing self from set 0x558e819c42b0 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e817e94a8 in 0x558e80219c00) _clear 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558e83d0f180 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d0f1f0 sbid 0x0)) 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x111000~2000] 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f180) discard_unallocated 0x0~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0f208 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x111000~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.sharedblob(0x558e83d0f1f0) put 0x558e83d0f1f0 removing self from set 0x558e819c42b0 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0f208 in 0x558e80219c00) _clear 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558e83d0f260 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x113000~2000] 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f260) discard_unallocated 0x0~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0f2e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x113000~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.sharedblob(0x558e83d0f2d0) put 0x558e83d0f2d0 removing self from set 0x558e819c42b0 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0f2e8 in 0x558e80219c00) _clear 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558e83d0f340 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d0f3b0 sbid 0x0)) 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x115000~2000] 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f340) discard_unallocated 0x0~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0f3c8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x115000~2000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.sharedblob(0x558e83d0f3b0) put 0x558e83d0f3b0 removing self from set 0x558e819c42b0 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0f3c8 in 0x558e80219c00) _clear 2019-12-07T19:33:58.823+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.bs3_head 3#1:d07269f7:::smithi13140796-28:head#c = 0 2019-12-07T19:33:58.823+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.823+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:33:58.823+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195fb00 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195fb00 onodes shared_blobs 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195fb00 allocated 0x[] released 0x[fb000~1c000] 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv release 0xfb000~1c000 2019-12-07T19:33:58.823+0000 7fe2c6263700 10 freelist release 0xfb000~1c000 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 freelist _xor first_key 0x80000 last_key 0x100000 2019-12-07T19:33:58.823+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 prepare 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195fb00 2019-12-07T19:33:58.823+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 io_done 2019-12-07T19:33:58.823+0000 7fe2c6263700 10 osd.0 21 dequeue_op 0x558e83b0b1e0 finish 2019-12-07T19:33:58.823+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.823+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 46 ==== MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=46, reqid=client.4540.0:163, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e819d1800 con 0x558e818d7000 2019-12-07T19:33:58.823+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83edcb00 prio 127 cost 0 latency 0.000025 epoch 21 MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=46, reqid=client.4540.0:163, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.823+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=46, reqid=client.4540.0:163, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1279 < min_flush_size 524288 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2000 had 1 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1302 < min_flush_size 524288 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1302 < min_flush_size 524288 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x60e0b~516 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x60e0b 0x60e0b~516 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x30e0b 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xe0b 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x321 and padding block with 0xcdf 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x61321 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000112939s (0.000000105s flush + 0.000112834s kv commit) 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.824+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.824+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195fb00> 2019-12-07T19:33:58.824+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.824+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 kv_submitted 2019-12-07T19:33:58.824+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195fb00 2019-12-07T19:33:58.824+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 finishing 2019-12-07T19:33:58.824+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195fb00 onodes 2019-12-07T19:33:58.824+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 done 2019-12-07T19:33:58.824+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:33:58.824+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195fb00 [fb000~1c000] 2019-12-07T19:33:58.824+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release 0xfb000~1c000 2019-12-07T19:33:58.824+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.824+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.824+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.bs0 21/18 ECSubWriteReply(tid=35, last_complete=21'12, committed=1, applied=1)) v2 -- 0x558e84029e00 con 0x558e818fe400 2019-12-07T19:33:58.824+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.824+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 87 ==== MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=88, to_read={1:9da56077:::smithi13140796-13:head=0,53248,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8402f400 con 0x558e818d7c00 2019-12-07T19:33:58.824+0000 7fe2e7d3c700 15 osd.0 21 enqueue_op 0x558e83edcdc0 prio 127 cost 0 latency 0.000022 epoch 21 MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=88, to_read={1:9da56077:::smithi13140796-13:head=0,53248,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.824+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=88, to_read={1:9da56077:::smithi13140796-13:head=0,53248,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) 2019-12-07T19:33:58.824+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=46, reqid=client.4540.0:163, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=46, reqid=client.4540.0:163, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2) prio 127 cost 0 e21) pg 0x558e819e4000 2019-12-07T19:33:58.824+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e83edcb00 prio 127 cost 0 latency 0.001419 MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=46, reqid=client.4540.0:163, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 pg pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] 2019-12-07T19:33:58.824+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] _handle_message: MOSDECSubOpWrite(1.1s1 21/18 ECSubWrite(tid=46, reqid=client.4540.0:163, at_version=0'0, trim_to=0'0, roll_forward_to=21'3)) v2 2019-12-07T19:33:58.824+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] append_log log((0'0,21'3], crt=21'2) [] 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 osd.0 pg_epoch: 21 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] rollforward: entry=21'3 (21'2) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:163 2019-12-07T19:33:58.819566+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:33:58.824+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log approx pg log length = 3 2019-12-07T19:33:58.824+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] append_log transaction_applied = 1 2019-12-07T19:33:58.824+0000 7fe2cb26d700 10 trim proposed trim_to = 0'0 2019-12-07T19:33:58.824+0000 7fe2cb26d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:33:58.824+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e8106c900 seq 10 2019-12-07T19:33:58.824+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:33:58.824+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:33:58.824+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c900 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c900 onodes shared_blobs 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c900 allocated 0x[] released 0x[] 2019-12-07T19:33:58.824+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 prepare 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c900 2019-12-07T19:33:58.824+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 io_done 2019-12-07T19:33:58.824+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e83edcb00 finish 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=88, to_read={1:9da56077:::smithi13140796-13:head=0,53248,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) queued 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=88, to_read={1:9da56077:::smithi13140796-13:head=0,53248,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e21) pg 0x558e8192b000 2019-12-07T19:33:58.824+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e83edcdc0 prio 127 cost 0 latency 0.000663 MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=88, to_read={1:9da56077:::smithi13140796-13:head=0,53248,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] 2019-12-07T19:33:58.824+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] _handle_message: MOSDECSubOpRead(1.9s3 21/18 ECSubRead(tid=88, to_read={1:9da56077:::smithi13140796-13:head=0,53248,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:33:58.824+0000 7fe2cb26d700 25 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:33:58.824+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x0~d000 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~d000 size 0xd000 (53248) 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866e70 blob([0x51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e0e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c642a0 blob([0x53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c641c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64150 blob([0x55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65110 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65180 blob([0x57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c651f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65260 blob([0x59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c652d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65340 blob([0x5b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83c653b0 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81866e70 blob([0x51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e0e0 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c64150 blob([0x55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65110 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c642a0 blob([0x53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c641c0 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c65180 blob([0x57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c651f0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c65260 blob([0x59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c652d0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x61321~471 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c65340 blob([0x5b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83c653b0 sbid 0x0)) need {<0x0, 0x1000> : [0xc000:0~1000]} 2019-12-07T19:33:58.824+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x61321 0x61321~471 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.824+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~1000 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x31321 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x321 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:58.824+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x792 and padding block with 0x86e 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:33:58.825+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x61792 2019-12-07T19:33:58.825+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:58.825+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:58.825+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:58.825+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:58.825+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000316876s (0.000000108s flush + 0.000316768s kv commit) 2019-12-07T19:33:58.825+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:58.825+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:58.825+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8106c900> 2019-12-07T19:33:58.825+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:58.825+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 kv_submitted 2019-12-07T19:33:58.825+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c900 2019-12-07T19:33:58.825+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 finishing 2019-12-07T19:33:58.825+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c900 onodes 2019-12-07T19:33:58.825+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c900 done 2019-12-07T19:33:58.825+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:33:58.825+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c900 [] 2019-12-07T19:33:58.825+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.825+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:33:58.825+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81866e70 blob([0x51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e0e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8194e0f8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e84032e10 space 0x558e8194e0f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c64150 blob([0x55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65110 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e83c65128 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e84032ea0 space 0x558e83c65128 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c642a0 blob([0x53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c641c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e83c641d8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81170e10 space 0x558e83c641d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e83c64da8 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81171c20 space 0x558e83c64da8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c65180 blob([0x57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c651f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e83c65208 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e84032f30 space 0x558e83c65208 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c65260 blob([0x59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c652d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e83c652e8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e84032fc0 space 0x558e83c652e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c65340 blob([0x5b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83c653b0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xc000:0~1000]} 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e83c653c8 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e84033050 space 0x558e83c653c8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:33:58.825+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x0~d000 = 53248 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read read request=53248 r=53248 len=53248 2019-12-07T19:33:58.825+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: Getting attr on 1:9da56077:::smithi13140796-13:head 2019-12-07T19:33:58.825+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: not in cache 1:9da56077:::smithi13140796-13:head 2019-12-07T19:33:58.825+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:33:58.825+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] get_hash_info: found on disk, size 53248 2019-12-07T19:33:58.825+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key 2019-12-07T19:33:58.825+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key = 0 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_sub_read: Checking hash of 1:9da56077:::smithi13140796-13:head 2019-12-07T19:33:58.825+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 21/18 ECSubReadReply(tid=88, attrs_read=0)) v2 -- 0x558e8191bb80 con 0x558e818d7c00 2019-12-07T19:33:58.825+0000 7fe2cb26d700 10 osd.0 21 dequeue_op 0x558e83edcdc0 finish 2019-12-07T19:33:58.825+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 21/18 ECSubWriteReply(tid=46, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e83fefe00 con 0x558e818d7000 2019-12-07T19:33:58.825+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:58.841+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.892+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.942+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:58.992+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.042+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.092+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.142+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.192+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.242+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.248+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:33:59.293+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.343+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.372+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.7 172.21.15.109:0/36939 6 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.690476875s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84029e00 con 0x558e81749000 2019-12-07T19:33:59.372+0000 7fe2e6d3a700 20 osd.0 21 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.689942191s,18446744073.690476875s]) 2019-12-07T19:33:59.372+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.827784287s delta_ub 0.019609425s) v5 -- 0x558e83fef800 con 0x558e81749000 2019-12-07T19:33:59.372+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.7 172.21.15.109:0/36939 6 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.808174862s delta_ub 18446744073.690476875s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84029e00 con 0x558e81748c00 2019-12-07T19:33:59.372+0000 7fe2e6d3a700 20 osd.0 21 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.689842256s,18446744073.690476875s]) 2019-12-07T19:33:59.372+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.372735+0000/22.808174862s send_stamp 22.827884222s delta_ub 0.019709360s) v5 -- 0x558e818a9600 con 0x558e81748c00 2019-12-07T19:33:59.393+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.443+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.450+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.1 172.21.15.131:0/37078 1 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.451422+0000/22.908345690s send_stamp 22.908345690s delta_ub 0.002539244s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84028000 con 0x558e81984c00 2019-12-07T19:33:59.450+0000 7fe2e6d3a700 20 osd.0 21 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002111492s,0.002539244s]) 2019-12-07T19:33:59.450+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.451422+0000/22.908345690s send_stamp 22.906234198s delta_ub 18446744073.707440124s) v5 -- 0x558e818a9600 con 0x558e81984c00 2019-12-07T19:33:59.450+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.1 172.21.15.131:0/37078 1 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.451422+0000/22.908345690s send_stamp 22.908345690s delta_ub 0.002539244s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83fefe00 con 0x558e81985000 2019-12-07T19:33:59.450+0000 7fe2e7d3c700 20 osd.0 21 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002089779s,0.002539244s]) 2019-12-07T19:33:59.450+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e21 up_from 13 ping_stamp 2019-12-07T19:33:59.451422+0000/22.908345690s send_stamp 22.906255911s delta_ub 18446744073.707461837s) v5 -- 0x558e81955c00 con 0x558e81985000 2019-12-07T19:33:59.493+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.543+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.551+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:33:59.593+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 290432 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.629+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 18 ==== osd_map(21..22 src has 1..22) v4 ==== 1230+0+0 (crc 0 0 0) 0x558e83dd06c0 con 0x558e80f39000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 osd.0 21 OSD::ms_dispatch: osd_map(21..22 src has 1..22) v4 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 osd.0 21 do_waiters -- start 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 osd.0 21 do_waiters -- finish 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 osd.0 21 _dispatch 0x558e83dd06c0 osd_map(21..22 src has 1..22) v4 2019-12-07T19:33:59.629+0000 7fe2d959f700 3 osd.0 21 handle_osd_map epochs [21,22], i have 21, src has [1,22] 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 osd.0 21 handle_osd_map got inc map for epoch 22 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 osd.0 21 got_full_map 22, nothing requested 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 osd.0 21 handle_osd_map pg_num_history pg_num_history(e22 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps purged_snaps {22={}} 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 osd.0 21 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e22 [1,22] lci=[13,22]) 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e811fa300 seq 14 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c89b972f:::inc_osdmap.22:0# key 0x7f7fffffffffffffffc89b97'/!inc_osdmap.22!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:59.629+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c89b972f:::inc_osdmap.22:0# 0x0~24d 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1107 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c89b972f:::inc_osdmap.22:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24d 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fa300 1 blobs 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0xfb000~1000/1000,1000,0 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0xfb000~1000] 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e8bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e8a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e8bd0 blob([0xfb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8a80 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.blob(0x558e817e8bd0) get_ref 0x0~24d Blob(0x558e817e8bd0 blob([0xfb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8a80 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.blob(0x558e817e8bd0) get_ref init 0x1000, 1000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558e817e8bd0 blob([0xfb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558e817e8a80 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e817e8a98 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x24d 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7c88) dirty_range mark inline shard dirty 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c89b972f:::inc_osdmap.22:0# 0x0~24d = 0 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3760e826:::osdmap.22:0# key 0x7f7fffffffffffffff3760e8'&!osdmap.22!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:33:59.629+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3760e826:::osdmap.22:0# 0x0~140c 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1108 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3760e826:::osdmap.22:0# 0x0~140c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~40c 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbf4 on front/back, now 0x0~1000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fa300 2 blobs 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0xfc000~2000/1000,2000,0 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0xfc000~2000] 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d02700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03730 blob([0xfc000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02700 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.blob(0x558e83d03730) get_ref 0x0~1000 Blob(0x558e83d03730 blob([0xfc000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02700 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.blob(0x558e83d03730) get_ref init 0x1000, 1000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e83d03730 blob([0xfc000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d02700 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e83d02718 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03880 blob([!~1000,0xfd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03810 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.blob(0x558e83d03880) get_ref 0x1000~40c Blob(0x558e83d03880 blob([!~1000,0xfd000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e83d03810 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.blob(0x558e83d03880) get_ref init 0x2000, 1000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~40c: 0x1000~40c Blob(0x558e83d03880 blob([!~1000,0xfd000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x558e83d03810 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e83d03828 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x140c 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28108) dirty_range mark inline shard dirty 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3760e826:::osdmap.22:0# 0x0~140c = 0 2019-12-07T19:33:59.629+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fa300 0 blobs 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:33:59.629+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:59.629+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:33:59.629+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fa300 0 blobs 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa300 cost 23202 (4 ios * 4000 + 7202 bytes) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa300 onodes 0x558e80251b80,0x558e817b9180,0x558e819b7b80,0x558e83f28000 shared_blobs 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7c88) update #-1:c89b972f:::inc_osdmap.22:0# 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.extentmap(0x558e819b7c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c89b972f:::inc_osdmap.22:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28108) update #-1:3760e826:::osdmap.22:0# 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28108) update inline shard 53 bytes from 2 extents 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3760e826:::osdmap.22:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa300 allocated 0x[fb000~3000] released 0x[] 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 freelist allocate 0xfb000~3000 2019-12-07T19:33:59.629+0000 7fe2d959f700 20 freelist _xor first_key 0x80000 last_key 0x80000 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 prepare 2019-12-07T19:33:59.629+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e811fa300 2019-12-07T19:33:59.629+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 aio_wait 2019-12-07T19:33:59.629+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa300 2019-12-07T19:33:59.629+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 io_done 2019-12-07T19:33:59.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:33:59.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:33:59.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:33:59.629+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:33:59.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:33:59.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:33:59.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b7b80 had 1 2019-12-07T19:33:59.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83f28000 had 1 2019-12-07T19:33:59.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:33:59.629+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3860398080 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:33:59.629+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998678) bluestore 86 GiB free (0.959969), bluefs_ratio 0.0399508 2019-12-07T19:33:59.629+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:33:59.629+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:33:59.629+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:59.629+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x61792~226c to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:59.629+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x61792 0x61792~226c to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:59.629+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:33:59.629+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x31792 2019-12-07T19:33:59.629+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x792 2019-12-07T19:33:59.629+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:33:59.629+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x9fe and padding block with 0x602 2019-12-07T19:33:59.630+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x639fe 2019-12-07T19:33:59.630+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:33:59.630+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:33:59.630+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:33:59.630+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:33:59.630+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000156119s (0.000005273s flush + 0.000150846s kv commit) 2019-12-07T19:33:59.630+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:33:59.630+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:33:59.630+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa300> 2019-12-07T19:33:59.630+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:33:59.630+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 kv_submitted 2019-12-07T19:33:59.630+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa300 2019-12-07T19:33:59.630+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e811fa300 osr 0x558e8107a500 2019-12-07T19:33:59.630+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e000) _discard drop 11 0x1b000~1000 2019-12-07T19:33:59.630+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e000) prepare_write seq 12 0x1b000~1000 crc 2e955191 2019-12-07T19:33:59.630+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e000) _discard drop 11 0x2000~1000 2019-12-07T19:33:59.630+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e000) prepare_write seq 12 0x2000~1000 crc 40dd6c45 2019-12-07T19:33:59.630+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0 21 _committed_osd_maps 22..22 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0 21 advance to epoch 22 (<= last 22 <= newest_map 22) 2019-12-07T19:33:59.630+0000 7fe2d7285700 7 osd.0 22 consume_map version 22 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 22 identify_splits_and_merges 1.9s3 e21 to e22 pg_nums {18=16} 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 22 identify_splits_and_merges 1.1s1 e21 to e22 pg_nums {18=16} 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 22 identify_splits_and_merges 1.as3 e21 to e22 pg_nums {18=16} 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 22 identify_splits_and_merges 1.2s0 e21 to e22 pg_nums {18=16} 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 22 identify_splits_and_merges 1.bs3 e21 to e22 pg_nums {18=16} 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 22 identify_splits_and_merges 1.4s1 e21 to e22 pg_nums {18=16} 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 22 identify_splits_and_merges 1.ds5 e21 to e22 pg_nums {18=16} 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 22 identify_splits_and_merges 1.es4 e21 to e22 pg_nums {18=16} 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 22 identify_splits_and_merges 1.7s3 e21 to e22 pg_nums {18=16} 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 22 identify_splits_and_merges 1.fs1 e21 to e22 pg_nums {18=16} 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 22 prune_pg_created 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0:0.consume_map 22 (was 21) 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0:1.consume_map 22 (was 21) 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0:2.consume_map 22 (was 21) 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0:3.consume_map 22 (was 21) 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0:4.consume_map 22 (was 21) 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0:5.consume_map 22 (was 21) 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0:6.consume_map 22 (was 21) 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0:7.consume_map 22 (was 21) 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0 22 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:33:59.630+0000 7fe2d7285700 15 osd.0 22 enqueue_peering_evt 1.9s3 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:33:59.630+0000 7fe2d7285700 15 osd.0 22 enqueue_peering_evt 1.1s1 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:33:59.630+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:33:59.630+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558e8192b000 2019-12-07T19:33:59.630+0000 7fe2d7285700 15 osd.0 22 enqueue_peering_evt 1.as3 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:33:59.630+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2d7285700 15 osd.0 22 enqueue_peering_evt 1.2s0 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:33:59.630+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 22 2019-12-07T19:33:59.630+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558e819e4000 2019-12-07T19:33:59.630+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:33:59.630+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.9s3 21 -> 22 2019-12-07T19:33:59.630+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558e8193b000 2019-12-07T19:33:59.630+0000 7fe2c7265700 10 osd.0 pg_epoch: 21 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:33:59.630+0000 7fe2c7265700 10 osd.0 pg_epoch: 22 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:33:59.630+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_advance_map: 22 2019-12-07T19:33:59.630+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.1s1 21 -> 22 2019-12-07T19:33:59.630+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_advance_map: 22 2019-12-07T19:33:59.630+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.as3 21 -> 22 2019-12-07T19:33:59.630+0000 7fe2cb26d700 10 osd.0 pg_epoch: 21 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_advance_map [3,0,6,4,5,1]/[3,0,6,4,5,1] -- 3/3 2019-12-07T19:33:59.630+0000 7fe2c7265700 10 osd.0 pg_epoch: 22 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:59.630+0000 7fe2c6a64700 10 osd.0 pg_epoch: 21 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_advance_map [1,6,2,0,5,4]/[1,6,2,0,5,4] -- 1/1 2019-12-07T19:33:59.630+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:33:59.630+0000 7fe2cb26d700 10 osd.0 pg_epoch: 22 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] state: Started advmap 2019-12-07T19:33:59.630+0000 7fe2c7265700 10 osd.0 pg_epoch: 22 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:59.630+0000 7fe2c6a64700 10 osd.0 pg_epoch: 22 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] state: Started advmap 2019-12-07T19:33:59.630+0000 7fe2cb26d700 10 osd.0 pg_epoch: 22 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:59.630+0000 7fe2c7265700 20 osd.0 pg_epoch: 22 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:33:59.630+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:33:59.630+0000 7fe2c6a64700 10 osd.0 pg_epoch: 22 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:59.630+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:33:59.630+0000 7fe2c7265700 20 osd.0 pg_epoch: 22 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:33:59.630+0000 7fe2cb26d700 10 osd.0 pg_epoch: 22 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:59.630+0000 7fe2c7265700 20 osd.0 22 advance_pg new pool opts old pool opts 2019-12-07T19:33:59.630+0000 7fe2c6a64700 10 osd.0 pg_epoch: 22 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:59.630+0000 7fe2cb26d700 20 osd.0 pg_epoch: 22 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] hit_set_clear 2019-12-07T19:33:59.630+0000 7fe2c6a64700 20 osd.0 pg_epoch: 22 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] hit_set_clear 2019-12-07T19:33:59.630+0000 7fe2c7265700 10 osd.0 pg_epoch: 22 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 22 2019-12-07T19:33:59.630+0000 7fe2cb26d700 20 osd.0 pg_epoch: 22 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] agent_stop 2019-12-07T19:33:59.630+0000 7fe2c6a64700 20 osd.0 pg_epoch: 22 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] agent_stop 2019-12-07T19:33:59.630+0000 7fe2cb26d700 20 osd.0 22 advance_pg new pool opts old pool opts 2019-12-07T19:33:59.630+0000 7fe2c6a64700 20 osd.0 22 advance_pg new pool opts old pool opts 2019-12-07T19:33:59.630+0000 7fe2c7265700 10 osd.0 pg_epoch: 22 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:33:59.630+0000 7fe2cb26d700 10 osd.0 pg_epoch: 22 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_activate_map: 22 2019-12-07T19:33:59.630+0000 7fe2c6a64700 10 osd.0 pg_epoch: 22 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_activate_map: 22 2019-12-07T19:33:59.630+0000 7fe2c7265700 20 osd.0 pg_epoch: 22 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:33:59.630+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2cb26d700 10 osd.0 pg_epoch: 22 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] activate_map 2019-12-07T19:33:59.630+0000 7fe2c6a64700 10 osd.0 pg_epoch: 22 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] activate_map 2019-12-07T19:33:59.630+0000 7fe2c7265700 10 osd.0 pg_epoch: 22 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2cb26d700 20 osd.0 pg_epoch: 22 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:33:59.630+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:59.630+0000 7fe2cb26d700 10 osd.0 pg_epoch: 22 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:33:59.630+0000 7fe2c6a64700 20 osd.0 pg_epoch: 22 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:33:59.630+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2c6a64700 10 osd.0 pg_epoch: 22 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:59.630+0000 7fe2d7285700 15 osd.0 22 enqueue_peering_evt 1.bs3 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:33:59.630+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:33:59.630+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558e81075000 2019-12-07T19:33:59.630+0000 7fe2d7285700 15 osd.0 22 enqueue_peering_evt 1.4s1 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:33:59.630+0000 7fe2d7285700 15 osd.0 22 enqueue_peering_evt 1.ds5 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:33:59.630+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:33:59.630+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558e81928000 2019-12-07T19:33:59.630+0000 7fe2d7285700 15 osd.0 22 enqueue_peering_evt 1.es4 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:33:59.630+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:33:59.630+0000 7fe2d7285700 15 osd.0 22 enqueue_peering_evt 1.7s3 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:33:59.630+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_advance_map: 22 2019-12-07T19:33:59.630+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558e81938000 2019-12-07T19:33:59.630+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.2s0 21 -> 22 2019-12-07T19:33:59.630+0000 7fe2d7285700 15 osd.0 22 enqueue_peering_evt 1.fs1 epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0 22 maybe_update_heartbeat_peers forcing update after 20.106562 seconds 2019-12-07T19:33:59.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 21 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_advance_map [0,4,1,6,5,3]/[0,4,1,6,5,3] -- 0/0 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0 22 reset_heartbeat_peers 2019-12-07T19:33:59.630+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map: 22 2019-12-07T19:33:59.630+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs3 21 -> 22 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0 22 maybe_update_heartbeat_peers updating 2019-12-07T19:33:59.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] state: Active advmap 2019-12-07T19:33:59.630+0000 7fe2ca26b700 10 osd.0 pg_epoch: 21 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map [4,6,2,0,1,5]/[4,6,2,0,1,5] -- 4/4 2019-12-07T19:33:59.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] state: Started advmap 2019-12-07T19:33:59.630+0000 7fe2ca26b700 10 osd.0 pg_epoch: 22 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] state: Started advmap 2019-12-07T19:33:59.630+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_advance_map: 22 2019-12-07T19:33:59.630+0000 7fe2c5a62700 20 osd.0:4.update_pg_epoch 1.4s1 21 -> 22 2019-12-07T19:33:59.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] check_recovery_sources no source osds () went down 2019-12-07T19:33:59.630+0000 7fe2ca26b700 10 osd.0 pg_epoch: 22 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:59.630+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:33:59.630+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs3_head options 2019-12-07T19:33:59.630+0000 7fe2c5a62700 10 osd.0 pg_epoch: 21 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0 22 adding neighbor peer osd.1 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0 22 adding neighbor peer osd.5 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0 22 adding neighbor peer osd.7 2019-12-07T19:33:59.630+0000 7fe2ca26b700 10 osd.0 pg_epoch: 22 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0 22 adding random peer osd.3 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0 22 adding random peer osd.4 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0 22 adding random peer osd.6 2019-12-07T19:33:59.630+0000 7fe2c5a62700 10 osd.0 pg_epoch: 22 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] state: Started advmap 2019-12-07T19:33:59.630+0000 7fe2d7285700 10 osd.0 22 maybe_update_heartbeat_peers 7 peers, extras 1,2,3,4,5,6,7 2019-12-07T19:33:59.630+0000 7fe2d7285700 7 osd.0 22 activate_map version 22 2019-12-07T19:33:59.630+0000 7fe2ca26b700 20 osd.0 pg_epoch: 22 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] hit_set_clear 2019-12-07T19:33:59.630+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:33:59.630+0000 7fe2c5a62700 10 osd.0 pg_epoch: 22 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:59.630+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2ca26b700 20 osd.0 pg_epoch: 22 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] agent_stop 2019-12-07T19:33:59.630+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558e819e7000 2019-12-07T19:33:59.630+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:33:59.630+0000 7fe2ca26b700 20 osd.0 22 advance_pg new pool opts old pool opts 2019-12-07T19:33:59.630+0000 7fe2cc26f700 20 osd.0 22 agent_entry empty queue 2019-12-07T19:33:59.630+0000 7fe2ca26b700 10 osd.0 pg_epoch: 22 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_activate_map: 22 2019-12-07T19:33:59.630+0000 7fe2c5a62700 10 osd.0 pg_epoch: 22 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:59.630+0000 7fe2ca26b700 10 osd.0 pg_epoch: 22 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map 2019-12-07T19:33:59.630+0000 7fe2c5a62700 20 osd.0 pg_epoch: 22 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] hit_set_clear 2019-12-07T19:33:59.630+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 22 2019-12-07T19:33:59.630+0000 7fe2c5261700 20 osd.0:5.update_pg_epoch 1.ds5 21 -> 22 2019-12-07T19:33:59.630+0000 7fe2ca26b700 20 osd.0 pg_epoch: 22 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:33:59.630+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2c5a62700 20 osd.0 pg_epoch: 22 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] agent_stop 2019-12-07T19:33:59.630+0000 7fe2c5a62700 20 osd.0 22 advance_pg new pool opts old pool opts 2019-12-07T19:33:59.630+0000 7fe2c5261700 10 osd.0 pg_epoch: 21 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:33:59.630+0000 7fe2ca26b700 10 osd.0 pg_epoch: 22 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:33:59.630+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:33:59.630+0000 7fe2c5a62700 10 osd.0 pg_epoch: 22 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_activate_map: 22 2019-12-07T19:33:59.630+0000 7fe2c5261700 10 osd.0 pg_epoch: 22 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:33:59.630+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] plpg_on_pool_change 2019-12-07T19:33:59.630+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558e8122e000 2019-12-07T19:33:59.630+0000 7fe2c5a62700 10 osd.0 pg_epoch: 22 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] activate_map 2019-12-07T19:33:59.630+0000 7fe2c5261700 10 osd.0 pg_epoch: 22 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:59.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] hit_set_clear 2019-12-07T19:33:59.630+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:33:59.630+0000 7fe2c5a62700 20 osd.0 pg_epoch: 22 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:33:59.630+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] agent_stop 2019-12-07T19:33:59.630+0000 7fe2c5261700 10 osd.0 pg_epoch: 22 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:59.630+0000 7fe2caa6c700 20 osd.0 22 advance_pg new pool opts old pool opts 2019-12-07T19:33:59.630+0000 7fe2c5a62700 10 osd.0 pg_epoch: 22 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2c5261700 20 osd.0 pg_epoch: 22 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:33:59.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_activate_map: 22 2019-12-07T19:33:59.630+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map: 22 2019-12-07T19:33:59.630+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:33:59.630+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.7s3 21 -> 22 2019-12-07T19:33:59.630+0000 7fe2c5261700 20 osd.0 pg_epoch: 22 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:33:59.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] activate_map 2019-12-07T19:33:59.630+0000 7fe2c5261700 20 osd.0 22 advance_pg new pool opts old pool opts 2019-12-07T19:33:59.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] state: Active: handling ActMap 2019-12-07T19:33:59.630+0000 7fe2c5261700 10 osd.0 pg_epoch: 22 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 22 2019-12-07T19:33:59.630+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:33:59.630+0000 7fe2c5261700 10 osd.0 pg_epoch: 22 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:33:59.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] Active: kicking snap trim 2019-12-07T19:33:59.630+0000 7fe2c425f700 10 osd.0 pg_epoch: 22 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] state: Started advmap 2019-12-07T19:33:59.630+0000 7fe2c5261700 20 osd.0 pg_epoch: 22 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:33:59.630+0000 7fe2caa6c700 7 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] state: handle ActMap primary 2019-12-07T19:33:59.630+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2c425f700 10 osd.0 pg_epoch: 22 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:59.630+0000 7fe2c5261700 10 osd.0 pg_epoch: 22 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:33:59.630+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:33:59.630+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:33:59.630+0000 7fe2caa6c700 15 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] publish_stats_to_osd 21:26 2019-12-07T19:33:59.630+0000 7fe2c425f700 10 osd.0 pg_epoch: 22 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:59.630+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:33:59.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:33:59.630+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2c425f700 20 osd.0 pg_epoch: 22 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] hit_set_clear 2019-12-07T19:33:59.630+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558e81a98000 2019-12-07T19:33:59.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2c425f700 20 osd.0 pg_epoch: 22 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] agent_stop 2019-12-07T19:33:59.630+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:33:59.630+0000 7fe2c425f700 20 osd.0 22 advance_pg new pool opts old pool opts 2019-12-07T19:33:59.630+0000 7fe2c425f700 10 osd.0 pg_epoch: 22 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_activate_map: 22 2019-12-07T19:33:59.630+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_advance_map: 22 2019-12-07T19:33:59.630+0000 7fe2c8a68700 20 osd.0:6.update_pg_epoch 1.es4 21 -> 22 2019-12-07T19:33:59.630+0000 7fe2c425f700 10 osd.0 pg_epoch: 22 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map 2019-12-07T19:33:59.630+0000 7fe2c8a68700 10 osd.0 pg_epoch: 21 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_advance_map [1,3,4,5,0,6]/[1,3,4,5,0,6] -- 1/1 2019-12-07T19:33:59.630+0000 7fe2c425f700 20 osd.0 pg_epoch: 22 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:33:59.630+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2c8a68700 10 osd.0 pg_epoch: 22 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] state: Started advmap 2019-12-07T19:33:59.630+0000 7fe2c8a68700 10 osd.0 pg_epoch: 22 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:59.630+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:33:59.630+0000 7fe2c425f700 10 osd.0 pg_epoch: 22 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2c8a68700 10 osd.0 pg_epoch: 22 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:59.630+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2c8a68700 20 osd.0 pg_epoch: 22 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] hit_set_clear 2019-12-07T19:33:59.630+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) queued 2019-12-07T19:33:59.630+0000 7fe2c8a68700 20 osd.0 pg_epoch: 22 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] agent_stop 2019-12-07T19:33:59.630+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:33:59.630+0000 7fe2c8a68700 20 osd.0 22 advance_pg new pool opts old pool opts 2019-12-07T19:33:59.630+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 22 epoch_requested: 22 NullEvt) prio 255 cost 10 e22) pg 0x558e81a9b000 2019-12-07T19:33:59.630+0000 7fe2c8a68700 10 osd.0 pg_epoch: 22 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_activate_map: 22 2019-12-07T19:33:59.630+0000 7fe2c8a68700 10 osd.0 pg_epoch: 22 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] activate_map 2019-12-07T19:33:59.630+0000 7fe2c8a68700 20 osd.0 pg_epoch: 22 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:33:59.630+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:33:59.630+0000 7fe2c8a68700 10 osd.0 pg_epoch: 22 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.630+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:33:59.631+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:33:59.630+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 22 2019-12-07T19:33:59.631+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.fs1 21 -> 22 2019-12-07T19:33:59.631+0000 7fe2c425f700 10 osd.0 pg_epoch: 21 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [3,0,5,4,1,6]/[3,0,5,4,1,6] -- 3/3 2019-12-07T19:33:59.631+0000 7fe2c425f700 10 osd.0 pg_epoch: 22 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:33:59.631+0000 7fe2c425f700 10 osd.0 pg_epoch: 22 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:33:59.631+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:33:59.631+0000 7fe2c425f700 10 osd.0 pg_epoch: 22 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:33:59.631+0000 7fe2c425f700 20 osd.0 pg_epoch: 22 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:33:59.631+0000 7fe2c425f700 20 osd.0 pg_epoch: 22 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:33:59.631+0000 7fe2c425f700 20 osd.0 22 advance_pg new pool opts old pool opts 2019-12-07T19:33:59.631+0000 7fe2c425f700 10 osd.0 pg_epoch: 22 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 22 2019-12-07T19:33:59.631+0000 7fe2c425f700 10 osd.0 pg_epoch: 22 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:33:59.631+0000 7fe2c425f700 20 osd.0 pg_epoch: 22 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 22 2019-12-07T19:33:59.631+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:33:59.631+0000 7fe2c425f700 10 osd.0 pg_epoch: 22 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 22 epoch_requested: 22 NullEvt 2019-12-07T19:33:59.631+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:33:59.631+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:33:59.643+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.685+0000 7fe2e2ca2700 10 osd.0 22 tick_without_osd_lock 2019-12-07T19:33:59.685+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fd32000/0x40000000/0x1680000000, data 0x2a7401/0x2cc000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:33:59.685+0000 7fe2e2ca2700 25 osd.0 22 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453695+0000 last_rx_front 2019-12-07T19:33:56.453704+0000 2019-12-07T19:33:59.685+0000 7fe2e2ca2700 25 osd.0 22 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453559+0000 last_rx_front 2019-12-07T19:33:56.453627+0000 2019-12-07T19:33:59.685+0000 7fe2e2ca2700 25 osd.0 22 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453575+0000 last_rx_front 2019-12-07T19:33:56.453596+0000 2019-12-07T19:33:59.685+0000 7fe2e2ca2700 25 osd.0 22 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453839+0000 last_rx_front 2019-12-07T19:33:56.453807+0000 2019-12-07T19:33:59.685+0000 7fe2e2ca2700 25 osd.0 22 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453763+0000 last_rx_front 2019-12-07T19:33:56.453773+0000 2019-12-07T19:33:59.685+0000 7fe2e2ca2700 25 osd.0 22 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453751+0000 last_rx_front 2019-12-07T19:33:56.453819+0000 2019-12-07T19:33:59.685+0000 7fe2e2ca2700 25 osd.0 22 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453789+0000 last_rx_front 2019-12-07T19:33:56.453733+0000 2019-12-07T19:33:59.685+0000 7fe2e2ca2700 20 osd.0 22 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:33:59.685+0000 7fe2e2ca2700 10 osd.0 22 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:33:59.685+0000 7fe2e2ca2700 20 osd.0 22 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:33:59.685+0000 7fe2e2ca2700 10 osd.0 22 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:33:59.693+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.708+0000 7fe2e4535700 10 osd.0 22 tick 2019-12-07T19:33:59.708+0000 7fe2e4535700 10 osd.0 22 do_waiters -- start 2019-12-07T19:33:59.708+0000 7fe2e4535700 10 osd.0 22 do_waiters -- finish 2019-12-07T19:33:59.708+0000 7fe2e4535700 20 osd.0 22 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:33:59.744+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.751+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.3 172.21.15.131:0/37076 2 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.752408+0000/23.209920745s send_stamp 23.209920745s delta_ub 0.002886275s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84029600 con 0x558e818ff800 2019-12-07T19:33:59.751+0000 7fe2e753b700 20 osd.0 22 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002569298s,0.002886275s]) 2019-12-07T19:33:59.751+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e22 up_from 13 ping_stamp 2019-12-07T19:33:59.752408+0000/23.209920745s send_stamp 23.207351447s delta_ub 18446744073.706982318s) v5 -- 0x558e8118f400 con 0x558e818ff800 2019-12-07T19:33:59.751+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.3 172.21.15.131:0/37076 2 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.752408+0000/23.209920745s send_stamp 23.209920745s delta_ub 0.002886275s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84029e00 con 0x558e81984000 2019-12-07T19:33:59.751+0000 7fe2e753b700 10 osd.0 22 send_incremental_map 21 -> 22 to 0x558e818d7000 v2:172.21.15.131:6812/37076 2019-12-07T19:33:59.752+0000 7fe2e753b700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c89b972f:::inc_osdmap.22:0# 0x0~0 2019-12-07T19:33:59.752+0000 7fe2e753b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~24d size 0x24d (589) 2019-12-07T19:33:59.752+0000 7fe2e753b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:33:59.752+0000 7fe2e753b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8bd0 blob([0xfb000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558e817e8a80 sbid 0x0)) need 0x0~24d cache has 0x[0~24d] 2019-12-07T19:33:59.752+0000 7fe2e753b700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c89b972f:::inc_osdmap.22:0# 0x0~24d = 589 2019-12-07T19:33:59.752+0000 7fe2e753b700 10 osd.0 22 add_map_inc_bl 22 589 bytes 2019-12-07T19:33:59.752+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(22..22 src has 1..22) v4 -- 0x558e83dcf8c0 con 0x558e818d7000 2019-12-07T19:33:59.752+0000 7fe2e753b700 10 osd.0 22 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 21 -> 22 (shared) 2019-12-07T19:33:59.752+0000 7fe2e6d3a700 20 osd.0 22 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002469933s,0.002886275s]) 2019-12-07T19:33:59.752+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e22 up_from 13 ping_stamp 2019-12-07T19:33:59.752408+0000/23.209920745s send_stamp 23.207450812s delta_ub 18446744073.707081683s) v5 -- 0x558e818a9600 con 0x558e81984000 2019-12-07T19:33:59.793+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.844+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.859+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.4 172.21.15.109:0/36938 1 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.859701+0000/23.308927630s send_stamp 23.308927630s delta_ub 18446744073.704378162s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83fefe00 con 0x558e81985800 2019-12-07T19:33:59.859+0000 7fe2e7d3c700 20 osd.0 22 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703685041s,18446744073.704378162s]) 2019-12-07T19:33:59.859+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e22 up_from 13 ping_stamp 2019-12-07T19:33:59.859701+0000/23.308927630s send_stamp 23.314794205s delta_ub 0.005866575s) v5 -- 0x558e81955c00 con 0x558e81985800 2019-12-07T19:33:59.859+0000 7fe2e7d3c700 10 osd.0 22 send_incremental_map 21 -> 22 to 0x558e818fe400 v2:172.21.15.109:6810/36938 2019-12-07T19:33:59.859+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.4 172.21.15.109:0/36938 1 ==== osd_ping(ping e21 up_from 13 ping_stamp 2019-12-07T19:33:59.859701+0000/23.308927630s send_stamp 23.308927630s delta_ub 18446744073.704378162s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84028000 con 0x558e81985400 2019-12-07T19:33:59.859+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(22..22 src has 1..22) v4 -- 0x558e83dcf8c0 con 0x558e818fe400 2019-12-07T19:33:59.859+0000 7fe2e7d3c700 10 osd.0 22 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 21 -> 22 (shared) 2019-12-07T19:33:59.859+0000 7fe2e6d3a700 20 osd.0 22 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703575056s,18446744073.704378162s]) 2019-12-07T19:33:59.859+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e22 up_from 13 ping_stamp 2019-12-07T19:33:59.859701+0000/23.308927630s send_stamp 23.314904190s delta_ub 0.005976560s) v5 -- 0x558e818a9600 con 0x558e81985400 2019-12-07T19:33:59.894+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.944+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:33:59.955+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:33:59.994+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.044+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.094+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.144+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.194+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.244+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.252+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.3 172.21.15.131:0/37076 3 ==== osd_ping(ping e22 up_from 13 ping_stamp 2019-12-07T19:34:00.252840+0000/23.710352893s send_stamp 23.710352893s delta_ub 0.003082049s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e819dec00 con 0x558e818ff800 2019-12-07T19:34:00.252+0000 7fe2e753b700 20 osd.0 22 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002687135s,0.003082049s]) 2019-12-07T19:34:00.252+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e22 up_from 13 ping_stamp 2019-12-07T19:34:00.252840+0000/23.710352893s send_stamp 23.707665758s delta_ub 18446744073.706864481s) v5 -- 0x558e8118f400 con 0x558e818ff800 2019-12-07T19:34:00.252+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.3 172.21.15.131:0/37076 3 ==== osd_ping(ping e22 up_from 13 ping_stamp 2019-12-07T19:34:00.252840+0000/23.710352893s send_stamp 23.710352893s delta_ub 0.003082049s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84029e00 con 0x558e81984000 2019-12-07T19:34:00.252+0000 7fe2e6d3a700 20 osd.0 22 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002657512s,0.003082049s]) 2019-12-07T19:34:00.252+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e22 up_from 13 ping_stamp 2019-12-07T19:34:00.252840+0000/23.710352893s send_stamp 23.707695381s delta_ub 18446744073.706894104s) v5 -- 0x558e818a9600 con 0x558e81984000 2019-12-07T19:34:00.294+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.345+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.364+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:00.395+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.442+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.5 172.21.15.109:0/36936 7 ==== osd_ping(ping e22 up_from 13 ping_stamp 2019-12-07T19:34:00.442978+0000/23.909301593s send_stamp 23.909301593s delta_ub 0.011913800s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e811f6600 con 0x558e8181b800 2019-12-07T19:34:00.442+0000 7fe2e7d3c700 20 osd.0 22 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011414706s,0.011913800s]) 2019-12-07T19:34:00.442+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e22 up_from 13 ping_stamp 2019-12-07T19:34:00.442978+0000/23.909301593s send_stamp 23.897886887s delta_ub 18446744073.698136910s) v5 -- 0x558e81955c00 con 0x558e8181b800 2019-12-07T19:34:00.442+0000 7fe2e7d3c700 10 osd.0 22 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 21 -> 22 (as per caller) 2019-12-07T19:34:00.442+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.5 172.21.15.109:0/36936 7 ==== osd_ping(ping e22 up_from 13 ping_stamp 2019-12-07T19:34:00.442978+0000/23.909301593s send_stamp 23.909301593s delta_ub 0.011913800s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83fefe00 con 0x558e8182c800 2019-12-07T19:34:00.442+0000 7fe2e7d3c700 20 osd.0 22 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011353582s,0.011913800s]) 2019-12-07T19:34:00.442+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e22 up_from 13 ping_stamp 2019-12-07T19:34:00.442978+0000/23.909301593s send_stamp 23.897948011s delta_ub 18446744073.698198034s) v5 -- 0x558e81955c00 con 0x558e8182c800 2019-12-07T19:34:00.445+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.456+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83d41800 0x558e81846100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:00.456+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83d41800 0x558e81846100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:00.456+0000 7fe2e6d3a700 10 osd.0 22 ms_handle_authentication new session 0x558e818e9a00 con 0x558e83d41800 entity client.admin addr 2019-12-07T19:34:00.456+0000 7fe2e6d3a700 10 osd.0 22 ms_handle_authentication session 0x558e818e9a00 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:00.456+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/890522420 conn(0x558e83d41800 0x558e81846100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=client.4352 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:00.456+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4352 172.21.15.131:0/890522420 1 ==== command(tid 1: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558e8122c780 con 0x558e83d41800 2019-12-07T19:34:00.456+0000 7fe2d959f700 20 osd.0 22 OSD::ms_dispatch: command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:00.456+0000 7fe2d959f700 10 osd.0 22 do_waiters -- start 2019-12-07T19:34:00.456+0000 7fe2d959f700 10 osd.0 22 do_waiters -- finish 2019-12-07T19:34:00.456+0000 7fe2d959f700 20 osd.0 22 _dispatch 0x558e8122c780 command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:00.457+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/890522420 -- command_reply(tid 1: 0 ) v1 -- 0x558e8022fdc0 con 0x558e83d41800 2019-12-07T19:34:00.480+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4352 172.21.15.131:0/890522420 2 ==== command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558e819c43c0 con 0x558e83d41800 2019-12-07T19:34:00.480+0000 7fe2d959f700 20 osd.0 22 OSD::ms_dispatch: command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:00.480+0000 7fe2d959f700 10 osd.0 22 do_waiters -- start 2019-12-07T19:34:00.480+0000 7fe2d959f700 10 osd.0 22 do_waiters -- finish 2019-12-07T19:34:00.480+0000 7fe2d959f700 20 osd.0 22 _dispatch 0x558e819c43c0 command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:00.482+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/890522420 -- command_reply(tid 2: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558e83c82700 con 0x558e83d41800 2019-12-07T19:34:00.495+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.521+0000 7fe2d5281700 20 osd.0 22 reports for 0 queries 2019-12-07T19:34:00.521+0000 7fe2d5281700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.0 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558e83ed6700 con 0x558e81748000 2019-12-07T19:34:00.521+0000 7fe2d5281700 20 bluestore(/var/lib/ceph/osd/ceph-0) pool_statfs pool 1 2019-12-07T19:34:00.521+0000 7fe2d5281700 10 bluestore(/var/lib/ceph/osd/ceph-0) pool_statfsstore_statfs(0x0/0x0/0x0, data 0x294000/0x294000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0) 2019-12-07T19:34:00.521+0000 7fe2d5281700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(1 pgs tid 0 v 0) v2 -- 0x558e83ed6a80 con 0x558e81748000 2019-12-07T19:34:00.545+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.560+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:00.595+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 1 osrs, 2 txcs 2019-12-07T19:34:00.595+0000 7fe2dc28f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock osr 0x558e8107a500 2 ios pending 2019-12-07T19:34:00.595+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 12 0x2000~1000 2019-12-07T19:34:00.595+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x2000~1000 crc 40dd6c45 2019-12-07T19:34:00.595+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 12 0x1b000~1000 2019-12-07T19:34:00.595+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x1b000~1000 crc 2e955191 2019-12-07T19:34:00.595+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.595+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish osr 0x558e8107a500 2019-12-07T19:34:00.595+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish dequeueing 2019-12-07T19:34:00.637+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 48 ==== pg_info2(1.bs3 1.bs3 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83c46e00 con 0x558e818fe400 2019-12-07T19:34:00.637+0000 7fe2e6d3a700 15 osd.0 22 enqueue_peering_evt 1.bs3 epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s) 2019-12-07T19:34:00.637+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.637+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.637+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.637+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.637+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e23) pg 0x558e81928000 2019-12-07T19:34:00.637+0000 7fe2c6263700 20 osd.0 op_wq _add_slot_waiter 1.bs3 peering, item epoch is 23, will wait on OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.637+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.639+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 47 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83ebb500 con 0x558e818d7000 2019-12-07T19:34:00.639+0000 7fe2e7d3c700 15 osd.0 22 enqueue_peering_evt 1.1s1 epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s) 2019-12-07T19:34:00.639+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.640+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.640+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.640+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.640+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e23) pg 0x558e819e4000 2019-12-07T19:34:00.640+0000 7fe2c7265700 20 osd.0 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 23, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.640+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.640+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 48 ==== pg_info2(1.fs1 1.fs1 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83ebb500 con 0x558e818d7000 2019-12-07T19:34:00.640+0000 7fe2e7d3c700 15 osd.0 22 enqueue_peering_evt 1.fs1 epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s) 2019-12-07T19:34:00.640+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 88 ==== pg_info2(1.9s3 1.9s3 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83ebb500 con 0x558e818d7c00 2019-12-07T19:34:00.641+0000 7fe2e7d3c700 15 osd.0 22 enqueue_peering_evt 1.9s3 epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:34:00.641+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.641+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e23) pg 0x558e81a9b000 2019-12-07T19:34:00.641+0000 7fe2c8267700 20 osd.0 op_wq _add_slot_waiter 1.fs1 peering, item epoch is 23, will wait on OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.641+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 89 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83ebb500 con 0x558e818d7c00 2019-12-07T19:34:00.641+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.641+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) pg 0x558e8192b000 2019-12-07T19:34:00.641+0000 7fe2cb26d700 20 osd.0 op_wq _add_slot_waiter 1.9s3 peering, item epoch is 23, will wait on OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.641+0000 7fe2e7d3c700 15 osd.0 22 enqueue_peering_evt 1.es4 epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s) 2019-12-07T19:34:00.641+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.641+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.641+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e23) pg 0x558e81a98000 2019-12-07T19:34:00.641+0000 7fe2c4a60700 20 osd.0 op_wq _add_slot_waiter 1.es4 peering, item epoch is 23, will wait on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.641+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.642+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 90 ==== pg_info2(1.7s3 1.7s3 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83ebb500 con 0x558e818d7c00 2019-12-07T19:34:00.642+0000 7fe2e7d3c700 15 osd.0 22 enqueue_peering_evt 1.7s3 epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:34:00.642+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.642+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.642+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.642+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.642+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) pg 0x558e8122e000 2019-12-07T19:34:00.642+0000 7fe2c425f700 20 osd.0 op_wq _add_slot_waiter 1.7s3 peering, item epoch is 23, will wait on OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.642+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.642+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 91 ==== pg_info2(1.as3 1.as3 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83ebb500 con 0x558e818d7c00 2019-12-07T19:34:00.642+0000 7fe2e7d3c700 15 osd.0 22 enqueue_peering_evt 1.as3 epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s) 2019-12-07T19:34:00.642+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.642+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.642+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.642+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.642+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e23) pg 0x558e8193b000 2019-12-07T19:34:00.642+0000 7fe2c6a64700 20 osd.0 op_wq _add_slot_waiter 1.as3 peering, item epoch is 23, will wait on OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.642+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.645+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 294068 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.651+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 14 ==== osd_op(client.4540.0:183 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e23) v8 ==== 264+0+0 (crc 0 0 0) 0x558e83da6840 con 0x558e83d68000 2019-12-07T19:34:00.651+0000 7fe2e6d3a700 15 osd.0 22 enqueue_op 0x558e83b0ac60 prio 63 cost 0 latency 0.000027 epoch 23 osd_op(client.4540.0:183 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e23) v8 2019-12-07T19:34:00.651+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:183 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e23) v8) prio 63 cost 0 e23) 2019-12-07T19:34:00.651+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.651+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:183 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e23) v8) prio 63 cost 0 e23) queued 2019-12-07T19:34:00.651+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.651+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:183 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e23) v8) prio 63 cost 0 e23) pg 0x558e81075000 2019-12-07T19:34:00.651+0000 7fe2caa6c700 10 osd.0 22 dequeue_op 0x558e83b0ac60 prio 63 cost 0 latency 0.000068 osd_op(client.4540.0:183 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e23) v8 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] 2019-12-07T19:34:00.651+0000 7fe2caa6c700 10 osd.0 22 maybe_share_map con 0x558e83d68000 172.21.15.131:0/224485236 map epoch 21 -> 23 (as per caller) 2019-12-07T19:34:00.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] do_request min 23, queue on waiting_for_map client.4540 2019-12-07T19:34:00.651+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- mon_subscribe({osdmap=23}) v3 -- 0x558e83fee200 con 0x558e80f39000 2019-12-07T19:34:00.651+0000 7fe2caa6c700 10 osd.0 22 dequeue_op 0x558e83b0ac60 finish 2019-12-07T19:34:00.651+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.651+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 19 ==== osd_map(23..23 src has 1..23) v4 ==== 661+0+0 (crc 0 0 0) 0x558e83d47200 con 0x558e80f39000 2019-12-07T19:34:00.651+0000 7fe2d959f700 20 osd.0 22 OSD::ms_dispatch: osd_map(23..23 src has 1..23) v4 2019-12-07T19:34:00.651+0000 7fe2d959f700 10 osd.0 22 do_waiters -- start 2019-12-07T19:34:00.651+0000 7fe2d959f700 10 osd.0 22 do_waiters -- finish 2019-12-07T19:34:00.651+0000 7fe2d959f700 20 osd.0 22 _dispatch 0x558e83d47200 osd_map(23..23 src has 1..23) v4 2019-12-07T19:34:00.651+0000 7fe2d959f700 3 osd.0 22 handle_osd_map epochs [23,23], i have 22, src has [1,23] 2019-12-07T19:34:00.651+0000 7fe2d959f700 10 osd.0 22 handle_osd_map got inc map for epoch 23 2019-12-07T19:34:00.651+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3760e826:::osdmap.22:0# 0x0~0 2019-12-07T19:34:00.651+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~140c size 0x140c (5132) 2019-12-07T19:34:00.651+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:00.651+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03730 blob([0xfc000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d02700 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:00.651+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03880 blob([!~1000,0xfd000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,40c]) SharedBlob(0x558e83d03810 sbid 0x0)) need 0x1000~40c cache has 0x[1000~40c] 2019-12-07T19:34:00.651+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3760e826:::osdmap.22:0# 0x0~140c = 5132 2019-12-07T19:34:00.651+0000 7fe2d959f700 10 osd.0 22 add_map_bl 22 5132 bytes 2019-12-07T19:34:00.651+0000 7fe2d959f700 20 osd.0 22 got_full_map 23, nothing requested 2019-12-07T19:34:00.651+0000 7fe2d959f700 20 osd.0 22 handle_osd_map pg_num_history pg_num_history(e23 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:00.651+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps purged_snaps {23={}} 2019-12-07T19:34:00.651+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:00.651+0000 7fe2d959f700 10 osd.0 22 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e23 [1,23] lci=[13,23]) 2019-12-07T19:34:00.651+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:00.651+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e81a0a900 seq 15 2019-12-07T19:34:00.651+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c57b972f:::inc_osdmap.23:0# key 0x7f7fffffffffffffffc57b97'/!inc_osdmap.23!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:00.651+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:00.651+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c57b972f:::inc_osdmap.23:0# 0x0~269 2019-12-07T19:34:00.651+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1109 2019-12-07T19:34:00.651+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c57b972f:::inc_osdmap.23:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:00.651+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.651+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~269 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e81a0a900 1 blobs 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0xfe000~1000/1000,1000,0 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0xfe000~1000] 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d02c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d02c40 blob([0xfe000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03c00 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.blob(0x558e83d02c40) get_ref 0x0~269 Blob(0x558e83d02c40 blob([0xfe000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03c00 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.blob(0x558e83d02c40) get_ref init 0x1000, 1000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558e83d02c40 blob([0xfe000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558e83d03c00 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e83d03c18 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x269 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28388) dirty_range mark inline shard dirty 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c57b972f:::inc_osdmap.23:0# 0x0~269 = 0 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3de0e826:::osdmap.23:0# key 0x7f7fffffffffffffff3de0e8'&!osdmap.23!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:00.652+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3de0e826:::osdmap.23:0# 0x0~1444 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1110 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3de0e826:::osdmap.23:0# 0x0~1444 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~444 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbbc on front/back, now 0x0~1000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e81a0a900 2 blobs 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0xff000~2000/1000,2000,0 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0xff000~2000] 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03ce0 blob([0xff000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03c70 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.blob(0x558e83d03ce0) get_ref 0x0~1000 Blob(0x558e83d03ce0 blob([0xff000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03c70 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.blob(0x558e83d03ce0) get_ref init 0x1000, 1000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e83d03ce0 blob([0xff000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d03c70 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e83d03c88 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03dc0 blob([!~1000,0x100000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03d50 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.blob(0x558e83d03dc0) get_ref 0x1000~444 Blob(0x558e83d03dc0 blob([!~1000,0x100000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e83d03d50 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.blob(0x558e83d03dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~444: 0x1000~444 Blob(0x558e83d03dc0 blob([!~1000,0x100000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,444]) SharedBlob(0x558e83d03d50 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e83d03d68 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1444 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28608) dirty_range mark inline shard dirty 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3de0e826:::osdmap.23:0# 0x0~1444 = 0 2019-12-07T19:34:00.652+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e81a0a900 0 blobs 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:00.652+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:00.652+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:00.652+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e81a0a900 0 blobs 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a900 cost 23286 (4 ios * 4000 + 7286 bytes) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a900 onodes 0x558e80251b80,0x558e817b9180,0x558e83f28280,0x558e83f28500 shared_blobs 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28388) update #-1:c57b972f:::inc_osdmap.23:0# 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28388) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c57b972f:::inc_osdmap.23:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28608) update #-1:3de0e826:::osdmap.23:0# 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28608) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3de0e826:::osdmap.23:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a900 allocated 0x[fe000~3000] released 0x[] 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 freelist allocate 0xfe000~3000 2019-12-07T19:34:00.652+0000 7fe2d959f700 20 freelist _xor first_key 0x80000 last_key 0x100000 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 prepare 2019-12-07T19:34:00.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e81a0a900 2019-12-07T19:34:00.652+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 aio_wait 2019-12-07T19:34:00.652+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a900 2019-12-07T19:34:00.652+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 io_done 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:00.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8818 < min_flush_size 524288 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83f28280 had 1 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83f28500 had 1 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:00.652+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3860398080 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:00.652+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998678) bluestore 86 GiB free (0.959969), bluefs_ratio 0.0399508 2019-12-07T19:34:00.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8865 < min_flush_size 524288 2019-12-07T19:34:00.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8865 < min_flush_size 524288 2019-12-07T19:34:00.652+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x639fe~22a1 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.652+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x639fe 0x639fe~22a1 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x339fe 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x9fe 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc9f and padding block with 0x361 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x65c9f 2019-12-07T19:34:00.652+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.652+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:00.652+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 1 in 0.000206704s (0.000006739s flush + 0.000199965s kv commit) 2019-12-07T19:34:00.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0a900> 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <0x558e83c6e000> 2019-12-07T19:34:00.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 kv_submitted 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a900 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e81a0a900 osr 0x558e8107a500 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) prepare_write seq 13 0x1b000~1000 crc 344afd6c 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) prepare_write seq 13 0x2000~1000 crc fbe67878 2019-12-07T19:34:00.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb200 deferred_cleanup 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fb200 onodes 0x558e80251b80,0x558e817b9180,0x558e819b7180,0x558e819b7400,0x558e819b7680,0x558e819b7900 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d0248 in 0x558e80219340) _finish_write discard buffer(0x558e80229a70 space 0x558e811d0248 0x0~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d0638 in 0x558e80219340) _finish_write discard buffer(0x558e80229ef0 space 0x558e811d0638 0x1000~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81867cf8 in 0x558e80219340) _finish_write discard buffer(0x558e81170480 space 0x558e81867cf8 0x0~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81867e48 in 0x558e80219340) _finish_write discard buffer(0x558e811705a0 space 0x558e81867e48 0x0~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81867f28 in 0x558e80219340) _finish_write discard buffer(0x558e81170510 space 0x558e81867f28 0x1000~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194ff28 in 0x558e80219340) _finish_write discard buffer(0x558e811707e0 space 0x558e8194ff28 0x0~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb200 done 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa300 deferred_cleanup 2019-12-07T19:34:00.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fb200 [] 2019-12-07T19:34:00.652+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:00.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 deferred_cleanup 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa300 onodes 0x558e80251b80,0x558e817b9180,0x558e819b7b80,0x558e83f28000 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e8a98 in 0x558e80219340) _finish_write discard buffer(0x558e811703f0 space 0x558e817e8a98 0x0~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7fe2d7285700 10 osd.0 22 _committed_osd_maps 23..23 2019-12-07T19:34:00.652+0000 7fe2d7285700 10 osd.0 22 advance to epoch 23 (<= last 23 <= newest_map 23) 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d02718 in 0x558e80219340) _finish_write discard buffer(0x558e811706c0 space 0x558e83d02718 0x0~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03828 in 0x558e80219340) _finish_write discard buffer(0x558e81170630 space 0x558e83d03828 0x1000~1000 writing nocache) 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa300 done 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a900 deferred_queued 2019-12-07T19:34:00.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa300 [] 2019-12-07T19:34:00.652+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:00.652+0000 7fe2d7285700 7 osd.0 23 consume_map version 23 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 23 identify_splits_and_merges 1.9s3 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 23 identify_splits_and_merges 1.1s1 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 23 identify_splits_and_merges 1.as3 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 23 identify_splits_and_merges 1.2s0 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 23 identify_splits_and_merges 1.bs3 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 23 identify_splits_and_merges 1.4s1 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 23 identify_splits_and_merges 1.ds5 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 23 identify_splits_and_merges 1.es4 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 23 identify_splits_and_merges 1.7s3 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 23 identify_splits_and_merges 1.fs1 e22 to e23 pg_nums {18=16} 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 23 prune_pg_created 2019-12-07T19:34:00.652+0000 7fe2d7285700 10 osd.0:0.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7fe2d7285700 10 osd.0:1.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.9s3 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.652+0000 7fe2d7285700 10 osd.0:2.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.as3 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:00.652+0000 7fe2d7285700 10 osd.0:3.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.bs3 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.652+0000 7fe2d7285700 10 osd.0:4.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:00.652+0000 7fe2d7285700 10 osd.0:5.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:00.652+0000 7fe2d7285700 10 osd.0:6.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.652+0000 7fe2d7285700 10 osd.0:7.consume_map 23 (was 22) 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.7s3 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 pending_peering first epoch 23 <= 23, requeueing 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.fs1 to_process <> waiting <> waiting_peering {23=} 2019-12-07T19:34:00.652+0000 7fe2d7285700 10 osd.0 23 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:00.652+0000 7fe2d7285700 15 osd.0 23 enqueue_peering_evt 1.9s3 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7fe2d7285700 15 osd.0 23 enqueue_peering_evt 1.1s1 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7fe2d7285700 15 osd.0 23 enqueue_peering_evt 1.as3 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7fe2d7285700 15 osd.0 23 enqueue_peering_evt 1.2s0 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.652+0000 7fe2d7285700 15 osd.0 23 enqueue_peering_evt 1.bs3 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s)) prio 255 cost 10 e23) pg 0x558e819e4000 2019-12-07T19:34:00.652+0000 7fe2d7285700 15 osd.0 23 enqueue_peering_evt 1.4s1 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.652+0000 7fe2d7285700 15 osd.0 23 enqueue_peering_evt 1.ds5 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s)) prio 255 cost 10 e23) pg 0x558e81a9b000 2019-12-07T19:34:00.652+0000 7fe2d7285700 15 osd.0 23 enqueue_peering_evt 1.es4 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7fe2d7285700 15 osd.0 23 enqueue_peering_evt 1.7s3 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.652+0000 7fe2d7285700 15 osd.0 23 enqueue_peering_evt 1.fs1 epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7fe2d7285700 7 osd.0 23 activate_map version 23 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558e81938000 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 osd.0 pg_epoch: 22 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.1s1 22 -> 23 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 osd.0 pg_epoch: 22 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_advance_map [3,0,6,4,5,1]/[3,0,6,4,5,1] -- 3/3 2019-12-07T19:34:00.652+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.652+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.652+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s)) prio 255 cost 10 e23) pg 0x558e81a98000 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] state: Started advmap 2019-12-07T19:34:00.652+0000 7fe2c8267700 10 osd.0 pg_epoch: 22 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.652+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.fs1 22 -> 23 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 22 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 20 osd.0:4.update_pg_epoch 1.4s1 22 -> 23 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.652+0000 7fe2c8267700 10 osd.0 pg_epoch: 22 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [3,0,5,4,1,6]/[3,0,5,4,1,6] -- 3/3 2019-12-07T19:34:00.652+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 22 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:00.652+0000 7fe2c8a68700 10 osd.0 pg_epoch: 22 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.652+0000 7fe2c8a68700 20 osd.0:6.update_pg_epoch 1.es4 22 -> 23 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.652+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 23 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] state: Started advmap 2019-12-07T19:34:00.652+0000 7fe2c8a68700 10 osd.0 pg_epoch: 22 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_advance_map [1,3,4,5,0,6]/[1,3,4,5,0,6] -- 1/1 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 osd.0 pg_epoch: 23 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] hit_set_clear 2019-12-07T19:34:00.652+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.652+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 osd.0 pg_epoch: 23 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] agent_stop 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 23 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.652+0000 7fe2c8a68700 10 osd.0 pg_epoch: 23 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] state: Started advmap 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 osd.0 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:34:00.652+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.652+0000 7fe2c8a68700 10 osd.0 pg_epoch: 23 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.652+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 23 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.652+0000 7fe2c8267700 20 osd.0 pg_epoch: 23 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] activate_map 2019-12-07T19:34:00.652+0000 7fe2c8a68700 10 osd.0 pg_epoch: 23 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 23 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] hit_set_clear 2019-12-07T19:34:00.652+0000 7fe2c8267700 20 osd.0 pg_epoch: 23 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:34:00.652+0000 7fe2c8267700 20 osd.0 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.652+0000 7fe2c8a68700 20 osd.0 pg_epoch: 23 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] hit_set_clear 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 osd.0 pg_epoch: 23 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 23 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] agent_stop 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 20 osd.0 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.652+0000 7fe2c8a68700 20 osd.0 pg_epoch: 23 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] agent_stop 2019-12-07T19:34:00.652+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.652+0000 7fe2c8a68700 20 osd.0 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074132637s ub 36.074132637s int 16s) 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 23 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.652+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:34:00.652+0000 7fe2c8a68700 10 osd.0 pg_epoch: 23 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 23 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] activate_map 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.652+0000 7fe2c8267700 20 osd.0 pg_epoch: 23 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.652+0000 7fe2c8a68700 10 osd.0 pg_epoch: 23 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] activate_map 2019-12-07T19:34:00.652+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 23 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7fe2c8a68700 20 osd.0 pg_epoch: 23 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.652+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074133487s ub 36.074133487s int 16s) 2019-12-07T19:34:00.652+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 23 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7fe2c8a68700 10 osd.0 pg_epoch: 23 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074554249s ub 36.074554249s int 16s) 2019-12-07T19:34:00.652+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.652+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:00.652+0000 7fe2c8a68700 10 osd.0 pg_epoch: 23 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e83c56c00 seq 11 2019-12-07T19:34:00.652+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:00.652+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:00.652+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:00.652+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e83d4c900 seq 17 2019-12-07T19:34:00.652+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:00.652+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e83d4cc00 seq 13 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:00.652+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56c00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56c00 onodes shared_blobs 2019-12-07T19:34:00.652+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56c00 allocated 0x[] released 0x[] 2019-12-07T19:34:00.652+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:00.652+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:00.652+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:34:00.652+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:00.652+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:00.652+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 prepare 2019-12-07T19:34:00.652+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:00.652+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:00.652+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56c00 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 io_done 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c900 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c900 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c900 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4cc00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4cc00 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4cc00 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 prepare 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c900 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 io_done 2019-12-07T19:34:00.653+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 prepare 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558e819e7000 2019-12-07T19:34:00.653+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4cc00 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s)) prio 255 cost 10 e23) pg 0x558e81928000 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s)) prio 255 cost 10 e23) pg 0x558e8192b000 2019-12-07T19:34:00.653+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 io_done 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s)) prio 255 cost 10 e23) pg 0x558e8122e000 2019-12-07T19:34:00.653+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558e81a98000 2019-12-07T19:34:00.653+0000 7fe2c9269700 10 osd.0 pg_epoch: 22 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 osd.0 pg_epoch: 22 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 osd.0 pg_epoch: 22 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 osd.0 pg_epoch: 22 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.ds5 22 -> 23 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.7s3 22 -> 23 2019-12-07T19:34:00.653+0000 7fe2c8a68700 10 osd.0 pg_epoch: 23 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs3 22 -> 23 2019-12-07T19:34:00.653+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:00.653+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7fe2c9269700 10 osd.0 pg_epoch: 22 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 osd.0 pg_epoch: 22 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_advance_map [4,6,2,0,1,5]/[4,6,2,0,1,5] -- 4/4 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 osd.0 pg_epoch: 22 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_advance_map [1,6,4,0,7,2]/[1,6,4,0,7,2] -- 1/1 2019-12-07T19:34:00.653+0000 7fe2cc26f700 20 osd.0 23 agent_entry empty queue 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7fe2c9269700 10 osd.0 pg_epoch: 23 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] state: Started advmap 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 osd.0 pg_epoch: 23 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] state: Started advmap 2019-12-07T19:34:00.653+0000 7fe2c9269700 10 osd.0 pg_epoch: 23 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:00.653+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:34:00.653+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 osd.0 pg_epoch: 23 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2236 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs3_head options 2019-12-07T19:34:00.653+0000 7fe2c9269700 10 osd.0 pg_epoch: 23 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 osd.0 pg_epoch: 23 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:00.653+0000 7fe2c9269700 20 osd.0 pg_epoch: 23 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 pg_epoch: 23 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] hit_set_clear 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3354 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 osd.0 pg_epoch: 23 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] hit_set_clear 2019-12-07T19:34:00.653+0000 7fe2c9269700 20 osd.0 pg_epoch: 23 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 pg_epoch: 23 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] agent_stop 2019-12-07T19:34:00.653+0000 7fe2c9269700 20 osd.0 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 osd.0 pg_epoch: 23 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] agent_stop 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3377 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3377 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 osd.0 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7fe2c9269700 10 osd.0 pg_epoch: 23 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 osd.0 pg_epoch: 23 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7fe2c9269700 10 osd.0 pg_epoch: 23 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x65c9f~d31 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 osd.0 pg_epoch: 23 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x65c9f 0x65c9f~d31 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 pg_epoch: 23 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.653+0000 7fe2c9269700 20 osd.0 pg_epoch: 23 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 osd.0 pg_epoch: 23 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x35c9f 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc9f 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x9d0 and padding block with 0x630 2019-12-07T19:34:00.653+0000 7fe2c9269700 10 osd.0 pg_epoch: 23 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075985074s ub 36.075985074s int 16s) 2019-12-07T19:34:00.653+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 osd.0 pg_epoch: 23 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.063561597s ub 36.063561597s int 16s) 2019-12-07T19:34:00.653+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 osd.0 pg_epoch: 23 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e83d4d200 seq 22 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:34:00.653+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e81a0b500 seq 23 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.9s3 22 -> 23 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:34:00.653+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4d200 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4d200 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4d200 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d200 prepare 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4d200 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 osd.0 pg_epoch: 22 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_advance_map [1,4,5,0,7,6]/[1,4,5,0,7,6] -- 1/1 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b500 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d200 io_done 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b500 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b500 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] state: Started advmap 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 prepare 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b500 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 io_done 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558e8122e000 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558e81928000 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 pg_epoch: 23 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] hit_set_clear 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 pg_epoch: 23 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] agent_stop 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558e81a9b000 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 osd.0 pg_epoch: 23 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 osd.0 pg_epoch: 23 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 pg_epoch: 23 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.074537935s ub 36.074537935s int 16s) 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83c56f00 seq 13 2019-12-07T19:34:00.653+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56f00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56f00 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56f00 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 prepare 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56f00 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 io_done 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558e8192b000 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558e819e4000 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 osd.0 pg_epoch: 23 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.653+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x669d0 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 3 cleaned 0 in 0.000281153s (0.000000098s flush + 0.000281055s kv commit) 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s)) prio 255 cost 10 e23) pg 0x558e8193b000 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2236 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 22 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.as3 22 -> 23 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3354 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3377 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 22 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_advance_map [1,6,2,0,5,4]/[1,6,2,0,5,4] -- 1/1 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3377 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] state: Started advmap 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x669d0~d31 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x669d0 0x669d0~d31 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x369d0 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x9d0 2019-12-07T19:34:00.653+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x701 and padding block with 0x8ff 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] hit_set_clear 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] agent_stop 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] activate_map 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.075332677s ub 36.075332677s int 16s) 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e811faf00 seq 9 2019-12-07T19:34:00.653+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811faf00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811faf00 onodes shared_blobs 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811faf00 allocated 0x[] released 0x[] 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 prepare 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811faf00 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 io_done 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558e8193b000 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x67701 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 3 cleaned 0 in 0.000186863s (0.000000072s flush + 0.000186791s kv commit) 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 NullEvt) prio 255 cost 10 e23) pg 0x558e81075000 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_advance_map: 23 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.2s0 22 -> 23 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56c00,0x558e83d4c900,0x558e83d4cc00,0x558e83d4d200,0x558e81a0b500,0x558e83c56f00> 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 kv_submitted 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56c00 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 22 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] handle_advance_map [0,4,1,6,5,3]/[0,4,1,6,5,3] -- 0/0 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 finishing 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56c00 onodes 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56c00 done 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56c00 [] 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x67701~475 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 kv_submitted 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c900 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x67701 0x67701~475 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean] state: Active advmap 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 finishing 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c900 onodes 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c900 done 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x37701 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c900 [] 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x701 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 kv_submitted 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] on_active_advmap new removed_snaps [2~1], snap_trimq now [2~1] 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4cc00 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb76 and padding block with 0x48a 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 finishing 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4cc00 onodes 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4cc00 done 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4cc00 [] 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] state: Started advmap 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d200 kv_submitted 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4d200 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d200 finishing 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4d200 onodes 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4d200 done 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4d200 [] 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 kv_submitted 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b500 2019-12-07T19:34:00.653+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 finishing 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b500 onodes 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b500 done 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b500 [] 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 kv_submitted 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56f00 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 finishing 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] plpg_on_pool_change 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56f00 onodes 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56f00 done 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56f00 [] 2019-12-07T19:34:00.653+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:00.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] hit_set_clear 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] agent_stop 2019-12-07T19:34:00.653+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 23 advance_pg new pool opts old pool opts 2019-12-07T19:34:00.653+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] handle_activate_map: 23 2019-12-07T19:34:00.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x67b76 2019-12-07T19:34:00.653+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] activate_map 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:00.653+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] state: Active: handling ActMap 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000104325s (0.000000061s flush + 0.000104264s kv commit) 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] Active: kicking snap trim 2019-12-07T19:34:00.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:00.653+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] exit NotTrimming 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] enter Trimming 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] enter Trimming/WaitReservation 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:00.653+0000 7fe2c6a64700 15 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] publish_stats_to_osd 21:27 2019-12-07T19:34:00.653+0000 7fe2c6a64700 7 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] state: handle ActMap primary 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:00.653+0000 7fe2c6a64700 15 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] publish_stats_to_osd 22:28 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] activate_map: Not dirtying info: last_persisted is 21 while current is 23 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] requeue_map_waiters client.4540 0x558e83b0ac60 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:183 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e23) v8) prio 63 cost 0 e23) 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] do_peering_event: epoch_sent: 23 epoch_requested: 23 NullEvt 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:183 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e23) v8) prio 63 cost 0 e23) queued 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:183 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e23) v8) prio 63 cost 0 e23) pg 0x558e81075000 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 23 dequeue_op 0x558e83b0ac60 prio 63 cost 0 latency 0.002713 osd_op(client.4540.0:183 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e23) v8 pg pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] _handle_message: osd_op(client.4540.0:183 1.2s0 1.32227932 (undecoded) ondisk+write+known_if_redirected e23) v8 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] do_op: op osd_op(client.4540.0:183 1.2s0 1:4c9e444c:::smithi13140796-48:head [stat,delete] snapc 2=[] ondisk+write+known_if_redirected e23) v8 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] do_op osd_op(client.4540.0:183 1.2s0 1:4c9e444c:::smithi13140796-48:head [stat,delete] snapc 2=[] ondisk+write+known_if_redirected e23) v8 may_write may_read -> write-ordered flags ondisk+write+known_if_redirected 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] get_object_context: found obc in cache: 0x558e83ff0000 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] get_object_context: 0x558e83ff0000 1:4c9e444c:::smithi13140796-48:head rwstate(none n=0 w=0) oi: 1:4c9e444c:::smithi13140796-48:head(21'9 client.4540.0:144 dirty|data_digest s 802816 uv 9 dd 18c6bbbb alloc_hint [0 0 0]) exists: 1 ssc: 0x558e81123d40 snapset: 0=[]:{} 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] find_object_context 1:4c9e444c:::smithi13140796-48:head @head oi=1:4c9e444c:::smithi13140796-48:head(21'9 client.4540.0:144 dirty|data_digest s 802816 uv 9 dd 18c6bbbb alloc_hint [0 0 0]) 2019-12-07T19:34:00.653+0000 7fe2c6a64700 25 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] do_op oi 1:4c9e444c:::smithi13140796-48:head(21'9 client.4540.0:144 dirty|data_digest s 802816 uv 9 dd 18c6bbbb alloc_hint [0 0 0]) 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] do_op obc obc(1:4c9e444c:::smithi13140796-48:head rwstate(excl n=1 w=0)) 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] execute_ctx 0x558e81228e00 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] execute_ctx 1:4c9e444c:::smithi13140796-48:head [stat,delete] ov 21'9 av 23'10 snapc 2=[] snapset 0=[]:{} 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] do_osd_op 1:4c9e444c:::smithi13140796-48:head [stat,delete] 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] do_osd_op stat 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] stat oi has 802816 2019-12-07T19:33:58.780144+0000 2019-12-07T19:34:00.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] do_osd_op delete 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] _delete_oid 1:4c9e444c:::smithi13140796-48:head whiteout=0 no_whiteout=0 try_no_whiteout=0 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] make_writeable 1:4c9e444c:::smithi13140796-48:head snapset=0=[]:{} snapc=2=[] 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] deletion, decrementing num_dirty and clearing flag 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] make_writeable 1:4c9e444c:::smithi13140796-48:head done, snapset=2=[]:{} 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] finish_ctx 1:4c9e444c:::smithi13140796-48:head 0x558e81228e00 op delete 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] finish_ctx object 1:4c9e444c:::smithi13140796-48:head marks clean_regions clean_offsets: [802816~18446744073708748799], clean_omap: 0, new_object: 0 2019-12-07T19:34:00.653+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] execute_ctx alloc reply 0x558e8402e780 result 0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] calc_trim_to_aggressive limit = 21'9 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] op order client.4540 tid 183 last was 144 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] new_repop rep_tid 15 on osd_op(client.4540.0:183 1.2s0 1:4c9e444c:::smithi13140796-48:head [stat out=16b,delete] snapc 2=[] ondisk+write+known_if_redirected e23) v8 2019-12-07T19:34:00.654+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:00.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811faf00> 2019-12-07T19:34:00.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] new_repop: repgather(0x558e83dcc240 0'0 rep_tid=15 committed?=0 r=0) 2019-12-07T19:34:00.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 kv_submitted 2019-12-07T19:34:00.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811faf00 2019-12-07T19:34:00.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 finishing 2019-12-07T19:34:00.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811faf00 onodes 2019-12-07T19:34:00.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811faf00 done 2019-12-07T19:34:00.654+0000 7fe2c6a64700 7 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] issue_repop rep_tid 15 o 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:00.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:00.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811faf00 [] 2019-12-07T19:34:00.654+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:00.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:00.654+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] submit_transaction: op Op(1:4c9e444c:::smithi13140796-48:head v=23'10 tt=0'0 tid=15 reqid=client.4540.0:183 client_op=osd_op(client.4540.0:183 1.2s0 1:4c9e444c:::smithi13140796-48:head [stat out=16b,delete] snapc 2=[] ondisk+write+known_if_redirected e23) roll_forward_to=21'9 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] get_hash_info: Getting attr on 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] get_hash_info: not in cache 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] get_hash_info: found on disk, size 200704 2019-12-07T19:34:00.654+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# hinfo_key 2019-12-07T19:34:00.654+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# hinfo_key = 0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] operator(): delete, setting projected size to 0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] operator(): 1:4c9e444c:::smithi13140796-48:head projected size 0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] start_rmw: Op(1:4c9e444c:::smithi13140796-48:head v=23'10 tt=0'0 tid=15 reqid=client.4540.0:183 client_op=osd_op(client.4540.0:183 1.2s0 1:4c9e444c:::smithi13140796-48:head [stat out=16b,delete] snapc 2=[] ondisk+write+known_if_redirected e23) roll_forward_to=21'9 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] try_state_to_reads: Op(1:4c9e444c:::smithi13140796-48:head v=23'10 tt=0'0 tid=15 reqid=client.4540.0:183 client_op=osd_op(client.4540.0:183 1.2s0 1:4c9e444c:::smithi13140796-48:head [stat out=16b,delete] snapc 2=[] ondisk+write+known_if_redirected e23) roll_forward_to=21'9 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] try_reads_to_commit: starting commit on Op(1:4c9e444c:::smithi13140796-48:head v=23'10 tt=0'0 tid=15 reqid=client.4540.0:183 client_op=osd_op(client.4540.0:183 1.2s0 1:4c9e444c:::smithi13140796-48:head [stat out=16b,delete] snapc 2=[] ondisk+write+known_if_redirected e23) roll_forward_to=21'9 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=4 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] generate_transactions: 1:4c9e444c:::smithi13140796-48:head, current size is 802816 buffers are {} 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] operator(): new_size start 0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] operator(): to_overwrite: {} 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] operator(): to_append: {} 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] operator(): 1:4c9e444c:::smithi13140796-48:head resetting hinfo to logical size 0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] try_reads_to_commit: written: {1:4c9e444c:::smithi13140796-48:head={}} 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] try_reads_to_commit: op: Op(1:4c9e444c:::smithi13140796-48:head v=23'10 tt=0'0 tid=15 reqid=client.4540.0:183 client_op=osd_op(client.4540.0:183 1.2s0 1:4c9e444c:::smithi13140796-48:head [stat out=16b,delete] snapc 2=[] ondisk+write+known_if_redirected e23) roll_forward_to=21'9 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] try_reads_to_commit: written_set: {1:4c9e444c:::smithi13140796-48:head=[]} 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] try_reads_to_commit: 1:4c9e444c:::smithi13140796-48:head,{} 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 23 send_incremental_map 21 -> 23 to 0x558e818d7c00 v2:172.21.15.131:6823/37078 2019-12-07T19:34:00.654+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c57b972f:::inc_osdmap.23:0# 0x0~0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~269 size 0x269 (617) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d02c40 blob([0xfe000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558e83d03c00 sbid 0x0)) need 0x0~269 cache has 0x[0~269] 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c57b972f:::inc_osdmap.23:0# 0x0~269 = 617 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 23 add_map_inc_bl 23 617 bytes 2019-12-07T19:34:00.654+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(22..23 src has 1..23) v4 -- 0x558e83dd1b00 con 0x558e818d7c00 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 23 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 21 -> 23 (shared) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 23/18 ECSubWrite(tid=15, reqid=client.4540.0:183, at_version=23'10, trim_to=0'0, roll_forward_to=21'9)) v2 -- 0x558e83acb800 con 0x558e818d7c00 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 23 send_incremental_map 22 -> 23 to 0x558e818d7000 v2:172.21.15.131:6812/37076 2019-12-07T19:34:00.654+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(23..23 src has 1..23) v4 -- 0x558e83dd0fc0 con 0x558e818d7000 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 23 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 22 -> 23 (shared) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 23/18 ECSubWrite(tid=15, reqid=client.4540.0:183, at_version=23'10, trim_to=0'0, roll_forward_to=21'9)) v2 -- 0x558e811b6000 con 0x558e818d7000 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 23 send_incremental_map 22 -> 23 to 0x558e818fe400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:00.654+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(23..23 src has 1..23) v4 -- 0x558e8123afc0 con 0x558e818fe400 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 23 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 22 -> 23 (shared) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 23/18 ECSubWrite(tid=15, reqid=client.4540.0:183, at_version=23'10, trim_to=0'0, roll_forward_to=21'9)) v2 -- 0x558e811b8000 con 0x558e818fe400 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 23 send_incremental_map 22 -> 23 to 0x558e818d6800 v2:172.21.15.109:6802/36936 2019-12-07T19:34:00.654+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(23..23 src has 1..23) v4 -- 0x558e8123b440 con 0x558e818d6800 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 23 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 22 -> 23 (shared) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s4 23/18 ECSubWrite(tid=15, reqid=client.4540.0:183, at_version=23'10, trim_to=0'0, roll_forward_to=21'9)) v2 -- 0x558e83aca000 con 0x558e818d6800 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 23 send_incremental_map 21 -> 23 to 0x558e818ffc00 v2:172.21.15.109:6818/36937 2019-12-07T19:34:00.654+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- osd_map(22..23 src has 1..23) v4 -- 0x558e819b2fc0 con 0x558e818ffc00 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 23 maybe_share_map con 0x558e818ffc00 v2:172.21.15.109:6818/36937 map epoch 21 -> 23 (shared) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=15, reqid=client.4540.0:183, at_version=23'10, trim_to=0'0, roll_forward_to=21'9)) v2 -- 0x558e83acb000 con 0x558e818ffc00 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 earliest_dup_version = 0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 trim 23'10 (21'9) delete 1:4c9e444c:::smithi13140796-48:head by client.4540.0:183 2019-12-07T19:34:00.651652+0000 0 ObjectCleanRegions clean_offsets: [802816~18446744073708748799], clean_omap: 0, new_object: 0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] append_log log((0'0,21'9], crt=21'9) [23'10 (21'9) delete 1:4c9e444c:::smithi13140796-48:head by client.4540.0:183 2019-12-07T19:34:00.651652+0000 0 ObjectCleanRegions clean_offsets: [802816~18446744073708748799], clean_omap: 0, new_object: 0] 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 lua=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] add_log_entry 23'10 (21'9) delete 1:4c9e444c:::smithi13140796-48:head by client.4540.0:183 2019-12-07T19:34:00.651652+0000 0 ObjectCleanRegions clean_offsets: [802816~18446744073708748799], clean_omap: 0, new_object: 0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 lua=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] append_log approx pg log length = 10 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 lua=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 23'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e811fbb00 seq 17 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:4c9e444c:::smithi13140796-48:head#a key 0x8080000000000000014c9e44'L!smithi13140796-48!='0xfffffffffffffffe000000000000000a'o' 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# -> 0#1:4c9e444c:::smithi13140796-48:head#a 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# -> 0#1:4c9e444c:::smithi13140796-48:head#a = 0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000023.00000000000000000010' <- 0000000023.00000000000000000010 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fbb00 cost 5822 (1 ios * 4000 + 1822 bytes) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fbb00 onodes 0x558e81a03900 shared_blobs 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e81a03a08) update 0#1:4c9e444c:::smithi13140796-48:head#a 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:4c9e444c:::smithi13140796-48:head#a is 966 (485 bytes onode + 2 bytes spanning blobs + 479 bytes inline extents) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fbb00 allocated 0x[] released 0x[] 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 prepare 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fbb00 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 io_done 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 lua=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] op_applied version 23'10 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] eval_repop repgather(0x558e83dcc240 23'10 rep_tid=15 committed?=0 r=0) 2019-12-07T19:34:00.654+0000 7fe2c6a64700 10 osd.0 23 dequeue_op 0x558e83b0ac60 finish 2019-12-07T19:34:00.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:00.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1162 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2483 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03900 had 1 2019-12-07T19:34:00.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:00.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3b80 had 1 2019-12-07T19:34:00.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2506 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2506 < min_flush_size 524288 2019-12-07T19:34:00.655+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x67b76~9ca to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.655+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x67b76 0x67b76~9ca to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.655+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.655+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x37b76 2019-12-07T19:34:00.655+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb76 2019-12-07T19:34:00.655+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.655+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x540 and padding block with 0xac0 2019-12-07T19:34:00.655+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x68540 2019-12-07T19:34:00.655+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.655+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:00.655+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:00.655+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:00.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000432288s (0.000000091s flush + 0.000432197s kv commit) 2019-12-07T19:34:00.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:00.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:00.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fbb00> 2019-12-07T19:34:00.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 kv_submitted 2019-12-07T19:34:00.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fbb00 2019-12-07T19:34:00.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 finishing 2019-12-07T19:34:00.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fbb00 onodes 0x558e81a03900 2019-12-07T19:34:00.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fbb00 done 2019-12-07T19:34:00.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:00.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fbb00 [] 2019-12-07T19:34:00.655+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:00.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:00.655+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.655+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7fe2cea74700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:00.656+0000 7fe2cea74700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] SnapTrimmer state: NotTrimming: trimming 2 2019-12-07T19:34:00.656+0000 7fe2cea74700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim_wait trimq=[2~1]] exit Trimming/WaitReservation 2019-12-07T19:34:00.656+0000 7fe2cea74700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:00.656+0000 7fe2cea74700 15 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] publish_stats_to_osd 23:29 2019-12-07T19:34:00.656+0000 7fe2cea74700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:00.656+0000 7fe2cea74700 10 osd.0 23 queueing pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean trimq=[2~1]] for snaptrim 2019-12-07T19:34:00.656+0000 7fe2cea74700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=23) prio 5 cost 1048576 e23) 2019-12-07T19:34:00.656+0000 7fe2cea74700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim trimq=[2~1]] prepare_stats_for_publish reporting purged_snaps [] 2019-12-07T19:34:00.656+0000 7fe2cea74700 15 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim trimq=[2~1]] publish_stats_to_osd 23:30 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=23) prio 5 cost 1048576 e23) queued 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=23) prio 5 cost 1048576 e23) pg 0x558e81075000 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim trimq=[2~1]] snap_trimmer posting 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim trimq=[2~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 2 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 0 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000002_.0_0000000000000001.2) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim trimq=[2~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim] SnapTrimmer state: adding snap 2 to purged_snaps 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim ps=[2~1]] SnapTrimmer state: purged_snaps now [2~1], snap_trimq now [] 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e811fa900 seq 18 2019-12-07T19:34:00.656+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa900 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa900 onodes shared_blobs 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa900 allocated 0x[] released 0x[] 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 prepare 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa900 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 io_done 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim ps=[2~1]] share_pg_info 2019-12-07T19:34:00.656+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x558e8122b100 con 0x558e818d7c00 2019-12-07T19:34:00.656+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x558e83c46000 con 0x558e818d7000 2019-12-07T19:34:00.656+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x558e83c48300 con 0x558e818fe400 2019-12-07T19:34:00.656+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s4 1.2s4 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x558e8122b800 con 0x558e818d6800 2019-12-07T19:34:00.656+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- pg_info2(1.2s3 1.2s3 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 -- 0x558e81229500 con 0x558e818ffc00 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim ps=[2~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean+snaptrim ps=[2~1]] exit Trimming 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:00.656+0000 7fe2caa6c700 15 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] publish_stats_to_osd 23:31 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] enter NotTrimming 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:00.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] snap_trimmer complete 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:00.656+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.656+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1094 < min_flush_size 524288 2019-12-07T19:34:00.656+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:00.656+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:00.656+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1117 < min_flush_size 524288 2019-12-07T19:34:00.656+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.656+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x68540~45d to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.656+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x68540 0x68540~45d to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.656+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.656+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x38540 2019-12-07T19:34:00.656+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x540 2019-12-07T19:34:00.656+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.656+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x99d and padding block with 0x663 2019-12-07T19:34:00.656+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x6899d 2019-12-07T19:34:00.656+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.656+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:00.656+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:00.656+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:00.656+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000315985s (0.000000080s flush + 0.000315905s kv commit) 2019-12-07T19:34:00.656+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:00.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:00.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa900> 2019-12-07T19:34:00.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.656+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 kv_submitted 2019-12-07T19:34:00.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa900 2019-12-07T19:34:00.656+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 finishing 2019-12-07T19:34:00.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa900 onodes 2019-12-07T19:34:00.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa900 done 2019-12-07T19:34:00.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:00.656+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa900 [] 2019-12-07T19:34:00.656+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:00.656+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:00.656+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.656+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.657+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:00.657+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 17 ==== MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e810fce00 con 0x558e818d6800 2019-12-07T19:34:00.657+0000 7fe2e753b700 15 osd.0 23 enqueue_op 0x558e83b0b8c0 prio 196 cost 0 latency 0.000016 epoch 23 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.657+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) 2019-12-07T19:34:00.657+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 12 ==== pg_info2(1.4s1 1.4s1 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83ebbc00 con 0x558e818fe000 2019-12-07T19:34:00.657+0000 7fe2e753b700 15 osd.0 23 enqueue_peering_evt 1.4s1 epoch_sent: 23 epoch_requested: 23 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s) 2019-12-07T19:34:00.657+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.657+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.657+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.657+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.657+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s)) prio 255 cost 10 e23) pg 0x558e81938000 2019-12-07T19:34:00.657+0000 7fe2c5a62700 10 osd.0 pg_epoch: 23 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.052429711s ub 36.052429711s int 16s) 2019-12-07T19:34:00.657+0000 7fe2c5a62700 10 osd.0 pg_epoch: 23 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.657+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:00.657+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:34:00.657+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e811fa600 seq 9 2019-12-07T19:34:00.657+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:34:00.657+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:00.657+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:00.657+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:00.657+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:34:00.657+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa600 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.657+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa600 onodes shared_blobs 2019-12-07T19:34:00.657+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa600 allocated 0x[] released 0x[] 2019-12-07T19:34:00.657+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 prepare 2019-12-07T19:34:00.657+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa600 2019-12-07T19:34:00.657+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 io_done 2019-12-07T19:34:00.657+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:00.657+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 21 ==== MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84029c00 con 0x558e818ffc00 2019-12-07T19:34:00.657+0000 7fe2e7d3c700 15 osd.0 23 enqueue_op 0x558e83edd8c0 prio 196 cost 0 latency 0.000020 epoch 23 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.657+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) 2019-12-07T19:34:00.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:00.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:00.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:34:00.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:00.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:00.657+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x6899d~475 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.657+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x6899d 0x6899d~475 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.657+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.657+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3899d 2019-12-07T19:34:00.657+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x99d 2019-12-07T19:34:00.657+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.657+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xe12 and padding block with 0x1ee 2019-12-07T19:34:00.657+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x68e12 2019-12-07T19:34:00.657+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.657+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:00.657+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:00.657+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:00.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000099429s (0.000000075s flush + 0.000099354s kv commit) 2019-12-07T19:34:00.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:00.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:00.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa600> 2019-12-07T19:34:00.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.657+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 kv_submitted 2019-12-07T19:34:00.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa600 2019-12-07T19:34:00.657+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 finishing 2019-12-07T19:34:00.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa600 onodes 2019-12-07T19:34:00.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa600 done 2019-12-07T19:34:00.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df900 q now empty 2019-12-07T19:34:00.657+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa600 [] 2019-12-07T19:34:00.657+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:00.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:00.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) queued 2019-12-07T19:34:00.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) pg 0x558e81075000 2019-12-07T19:34:00.657+0000 7fe2c6a64700 10 osd.0 23 dequeue_op 0x558e83b0b8c0 prio 196 cost 0 latency 0.000413 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] 2019-12-07T19:34:00.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.657+0000 7fe2c6a64700 10 osd.0 23 dequeue_op 0x558e83b0b8c0 finish 2019-12-07T19:34:00.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) queued 2019-12-07T19:34:00.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) pg 0x558e81075000 2019-12-07T19:34:00.657+0000 7fe2c6a64700 10 osd.0 23 dequeue_op 0x558e83edd8c0 prio 196 cost 0 latency 0.000228 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] 2019-12-07T19:34:00.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.657+0000 7fe2c6a64700 10 osd.0 23 dequeue_op 0x558e83edd8c0 finish 2019-12-07T19:34:00.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.657+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:00.657+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:00.658+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 92 ==== MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83feec00 con 0x558e818d7c00 2019-12-07T19:34:00.658+0000 7fe2e7d3c700 15 osd.0 23 enqueue_op 0x558e83edda20 prio 196 cost 0 latency 0.000019 epoch 23 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.658+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) 2019-12-07T19:34:00.658+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.658+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) queued 2019-12-07T19:34:00.658+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.658+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) pg 0x558e81075000 2019-12-07T19:34:00.658+0000 7fe2caa6c700 10 osd.0 23 dequeue_op 0x558e83edda20 prio 196 cost 0 latency 0.000065 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] 2019-12-07T19:34:00.658+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.658+0000 7fe2caa6c700 10 osd.0 23 dequeue_op 0x558e83edda20 finish 2019-12-07T19:34:00.658+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.659+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 20 ==== pg_info2(1.ds5 1.ds5 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e23/23) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8122b800 con 0x558e818d6000 2019-12-07T19:34:00.659+0000 7fe2e753b700 15 osd.0 23 enqueue_peering_evt 1.ds5 epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s) 2019-12-07T19:34:00.659+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e23) 2019-12-07T19:34:00.659+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.659+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e23) queued 2019-12-07T19:34:00.659+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.659+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s)) prio 255 cost 10 e23) pg 0x558e819e7000 2019-12-07T19:34:00.659+0000 7fe2c5261700 10 osd.0 pg_epoch: 23 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] do_peering_event: epoch_sent: 23 epoch_requested: 23 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 36.088857038s ub 36.088857038s int 16s) 2019-12-07T19:34:00.659+0000 7fe2c5261700 10 osd.0 pg_epoch: 23 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={}] proc_primary_info updating purged_snaps to [2~1] 2019-12-07T19:34:00.659+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:00.659+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:34:00.659+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e819a1500 seq 16 2019-12-07T19:34:00.659+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:34:00.659+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:00.659+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:00.659+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:00.659+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:34:00.659+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1500 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:00.659+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1500 onodes shared_blobs 2019-12-07T19:34:00.659+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1500 allocated 0x[] released 0x[] 2019-12-07T19:34:00.659+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 prepare 2019-12-07T19:34:00.659+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1500 2019-12-07T19:34:00.659+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 io_done 2019-12-07T19:34:00.659+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:00.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:00.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:34:00.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:00.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:00.659+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x68e12~475 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.659+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x68e12 0x68e12~475 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.659+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.659+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x38e12 2019-12-07T19:34:00.659+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xe12 2019-12-07T19:34:00.659+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.659+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x287 and padding block with 0xd79 2019-12-07T19:34:00.659+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x69287 2019-12-07T19:34:00.659+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.659+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:00.659+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:00.659+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:00.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000245486s (0.000000106s flush + 0.000245380s kv commit) 2019-12-07T19:34:00.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:00.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:00.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1500> 2019-12-07T19:34:00.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.659+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 kv_submitted 2019-12-07T19:34:00.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1500 2019-12-07T19:34:00.659+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 finishing 2019-12-07T19:34:00.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1500 onodes 2019-12-07T19:34:00.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1500 done 2019-12-07T19:34:00.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:34:00.659+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1500 [] 2019-12-07T19:34:00.659+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:00.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:00.661+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 49 ==== MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818a9200 con 0x558e818d7000 2019-12-07T19:34:00.661+0000 7fe2e7d3c700 15 osd.0 23 enqueue_op 0x558e83eddb80 prio 196 cost 0 latency 0.000020 epoch 23 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.661+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) 2019-12-07T19:34:00.661+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.661+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) queued 2019-12-07T19:34:00.661+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.661+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) pg 0x558e81075000 2019-12-07T19:34:00.661+0000 7fe2c6a64700 10 osd.0 23 dequeue_op 0x558e83eddb80 prio 196 cost 0 latency 0.000099 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] 2019-12-07T19:34:00.661+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.661+0000 7fe2c6a64700 10 osd.0 23 dequeue_op 0x558e83eddb80 finish 2019-12-07T19:34:00.661+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.661+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.662+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:00.662+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 49 ==== MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818a9600 con 0x558e818fe400 2019-12-07T19:34:00.662+0000 7fe2e6d3a700 15 osd.0 23 enqueue_op 0x558e83b0b080 prio 196 cost 0 latency 0.000026 epoch 23 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.662+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) queued 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) pg 0x558e81075000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 osd.0 23 dequeue_op 0x558e83b0b080 prio 196 cost 0 latency 0.000064 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=15, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] handle_sub_write_reply Calling on_all_commit on Op(1:4c9e444c:::smithi13140796-48:head v=23'10 tt=0'0 tid=15 reqid=client.4540.0:183 client_op=osd_op(client.4540.0:183 1.2s0 1:4c9e444c:::smithi13140796-48:head [stat out=16b,delete] snapc 2=[] ondisk+write+known_if_redirected e23) roll_forward_to=21'9 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 luod=21'9 crt=21'9 lcod 21'8 mlcod 21'8 active+clean ps=[2~1]] repop_all_committed: repop tid 15 all committed 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] eval_repop repgather(0x558e83dcc240 23'10 rep_tid=15 committed?=1 r=0) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] commit: repgather(0x558e83dcc240 23'10 rep_tid=15 committed?=1 r=0) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 15 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] log_op_stats osd_op(client.4540.0:183 1.2s0 1:4c9e444c:::smithi13140796-48:head [stat out=16b,delete] snapc 2=[] ondisk+write+known_if_redirected e23) v8 inb 0 outb 0 lat 0.011229 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] sending reply on osd_op(client.4540.0:183 1.2s0 1:4c9e444c:::smithi13140796-48:head [stat out=16b,delete] snapc 2=[] ondisk+write+known_if_redirected e23) v8 0x558e8402e780 2019-12-07T19:34:00.662+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(183 smithi13140796-48 [stat,delete] v23'10 uv10 ondisk = 0) v8 -- 0x558e8402e780 con 0x558e83d68000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 15 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] publish_stats_to_osd 23:32 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] removing repgather(0x558e83dcc240 23'10 rep_tid=15 committed?=1 r=0) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] q front is repgather(0x558e83dcc240 23'10 rep_tid=15 committed?=1 r=0) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 15 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] remove_repop repgather(0x558e83dcc240 23'10 rep_tid=15 committed?=1 r=0) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] try_finish_rmw: Op(1:4c9e444c:::smithi13140796-48:head v=23'10 tt=0'0 tid=15 reqid=client.4540.0:183 client_op=osd_op(client.4540.0:183 1.2s0 1:4c9e444c:::smithi13140796-48:head [stat out=16b,delete] snapc 2=[] ondisk+write+known_if_redirected e23) roll_forward_to=21'9 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:4c9e444c:::smithi13140796-48:head=[]}) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] try_reads_to_commit: starting commit on Op(1:4c9e444c:::smithi13140796-48:head v=0'0 tt=0'0 tid=16 reqid=client.4540.0:183 roll_forward_to=23'10 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] try_reads_to_commit: op: Op(1:4c9e444c:::smithi13140796-48:head v=0'0 tt=0'0 tid=16 reqid=client.4540.0:183 roll_forward_to=23'10 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:00.662+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 23/18 ECSubWrite(tid=16, reqid=client.4540.0:183, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 -- 0x558e819d0000 con 0x558e818d7c00 2019-12-07T19:34:00.662+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 23/18 ECSubWrite(tid=16, reqid=client.4540.0:183, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 -- 0x558e819d1800 con 0x558e818d7000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 23/18 ECSubWrite(tid=16, reqid=client.4540.0:183, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 -- 0x558e83e54800 con 0x558e818fe400 2019-12-07T19:34:00.662+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s4 23/18 ECSubWrite(tid=16, reqid=client.4540.0:183, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 -- 0x558e83e57000 con 0x558e818d6800 2019-12-07T19:34:00.662+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- MOSDECSubOpWrite(1.2s3 23/18 ECSubWrite(tid=16, reqid=client.4540.0:183, at_version=0'0, trim_to=0'0, roll_forward_to=23'10)) v2 -- 0x558e83e57800 con 0x558e818ffc00 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=21'9 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] append_log log((0'0,23'10], crt=21'9) [] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] rollforward: entry=23'10 (21'9) delete 1:4c9e444c:::smithi13140796-48:head by client.4540.0:183 2019-12-07T19:34:00.651652+0000 0 ObjectCleanRegions clean_offsets: [802816~18446744073708748799], clean_omap: 0, new_object: 0 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] append_log approx pg log length = 10 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:00.662+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e811fb500 seq 19 2019-12-07T19:34:00.662+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head#a onode 0x558e81a03900 txc 0x558e811fb500 2019-12-07T19:34:00.662+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head#a 0x0 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83edee70) put_ref 0x0~2000 Blob(0x558e83edee70 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edefc0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd2f50) put_ref 0x0~2000 Blob(0x558e83dd2f50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edf730 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fb90) put_ref 0x0~2000 Blob(0x558e83e6fb90 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6fa40 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fc00) put_ref 0x0~2000 Blob(0x558e83e6fc00 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6fab0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fc70) put_ref 0x0~2000 Blob(0x558e83e6fc70 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6fb20 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fe30) put_ref 0x0~2000 Blob(0x558e83e6fe30 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6fce0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fea0) put_ref 0x0~2000 Blob(0x558e83e6fea0 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6fd50 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6ff10) put_ref 0x0~2000 Blob(0x558e83e6ff10 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6fdc0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6ff80) put_ref 0x0~2000 Blob(0x558e83e6ff80 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48000 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48070) put_ref 0x0~2000 Blob(0x558e83f48070 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f480e0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48150) put_ref 0x0~2000 Blob(0x558e83f48150 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f481c0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48230) put_ref 0x0~2000 Blob(0x558e83f48230 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f482a0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48310) put_ref 0x0~2000 Blob(0x558e83f48310 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48380 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f489a0) put_ref 0x0~2000 Blob(0x558e83f489a0 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edfa40 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48a10) put_ref 0x0~2000 Blob(0x558e83f48a10 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48a80 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49490) put_ref 0x0~2000 Blob(0x558e83f49490 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f492d0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49420) put_ref 0x0~2000 Blob(0x558e83f49420 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f49260 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f493b0) put_ref 0x0~2000 Blob(0x558e83f493b0 blob([0x295000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f491f0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49340) put_ref 0x0~2000 Blob(0x558e83f49340 blob([0x297000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f49030 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49180) put_ref 0x0~2000 Blob(0x558e83f49180 blob([0x299000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48fc0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49110) put_ref 0x0~2000 Blob(0x558e83f49110 blob([0x29b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48f50 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f490a0) put_ref 0x0~2000 Blob(0x558e83f490a0 blob([0x29d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48ee0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48e70) put_ref 0x0~2000 Blob(0x558e83f48e70 blob([0x29f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f49500 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49570) put_ref 0x0~2000 Blob(0x558e83f49570 blob([0x2a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f495e0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49650) put_ref 0x0~1000 Blob(0x558e83f49650 blob([0x2a3000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f496c0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e81a03a08) dirty_range mark inline shard dirty 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e83edee70 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83edefc0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x273000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83edee70) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83edefd8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 _rm buffer(0x558e83d118c0 space 0x558e83edefd8 0x0~1000 clean) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x273000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83edefc0) put 0x558e83edefc0 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83edefd8 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e83dd2f50 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83edf730 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x275000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83dd2f50) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83edf748 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x275000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83edf730) put 0x558e83edf730 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83edf748 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e83e6fb90 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6fa40 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x277000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fb90) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fa58 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x277000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83e6fa40) put 0x558e83e6fa40 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fa58 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e83e6fc00 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6fab0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x279000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fc00) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fac8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x279000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83e6fab0) put 0x558e83e6fab0 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fac8 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e83e6fc70 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6fb20 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x27b000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fc70) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fb38 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x27b000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83e6fb20) put 0x558e83e6fb20 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fb38 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e83e6fe30 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6fce0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x27d000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fe30) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fcf8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x27d000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83e6fce0) put 0x558e83e6fce0 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fcf8 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e83e6fea0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6fd50 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x27f000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fea0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fd68 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x27f000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83e6fd50) put 0x558e83e6fd50 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fd68 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558e83e6ff10 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6fdc0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x281000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6ff10) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fdd8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x281000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83e6fdc0) put 0x558e83e6fdc0 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83e6fdd8 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558e83e6ff80 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f48000 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x283000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6ff80) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48018 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x283000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f48000) put 0x558e83f48000 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48018 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558e83f48070 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f480e0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x285000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48070) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f480f8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x285000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f480e0) put 0x558e83f480e0 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f480f8 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558e83f48150 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f481c0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x287000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48150) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f481d8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x287000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f481c0) put 0x558e83f481c0 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f481d8 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558e83f48230 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f482a0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x289000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48230) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f482b8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x289000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f482a0) put 0x558e83f482a0 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f482b8 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558e83f48310 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f48380 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x28b000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48310) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48398 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x28b000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f48380) put 0x558e83f48380 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48398 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558e83f489a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83edfa40 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x28d000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f489a0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83edfa58 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x28d000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83edfa40) put 0x558e83edfa40 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83edfa58 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558e83f48a10 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f48a80 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x28f000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48a10) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48a98 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x28f000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f48a80) put 0x558e83f48a80 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48a98 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558e83f49490 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f492d0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x291000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49490) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f492e8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x291000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f492d0) put 0x558e83f492d0 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f492e8 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558e83f49420 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f49260 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x293000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49420) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f49278 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x293000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f49260) put 0x558e83f49260 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f49278 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558e83f493b0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f491f0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x295000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f493b0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f49208 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x295000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f491f0) put 0x558e83f491f0 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f49208 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558e83f49340 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f49030 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x297000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49340) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f49048 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x297000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f49030) put 0x558e83f49030 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f49048 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558e83f49180 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f48fc0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x299000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49180) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48fd8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x299000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f48fc0) put 0x558e83f48fc0 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48fd8 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558e83f49110 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f48f50 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x29b000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49110) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48f68 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x29b000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f48f50) put 0x558e83f48f50 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48f68 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558e83f490a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f48ee0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x29d000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f490a0) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48ef8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x29d000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f48ee0) put 0x558e83f48ee0 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f48ef8 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558e83f48e70 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f49500 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x29f000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f48e70) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f49518 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x29f000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f49500) put 0x558e83f49500 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f49518 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558e83f49570 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f495e0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2a1000~2000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49570) discard_unallocated 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f495f8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2a1000~2000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f495e0) put 0x558e83f495e0 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f495f8 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x30000~1000: 0x0~1000 Blob(0x558e83f49650 blob([!~1000] csum crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558e83f496c0 sbid 0x0)) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2a3000~1000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f49650) discard_unallocated 0x0~1000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f496d8 in 0x558e802195e0) _discard 0x0~1000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2a3000~1000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.sharedblob(0x558e83f496c0) put 0x558e83f496c0 removing self from set 0x558e819b1390 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83f496d8 in 0x558e802195e0) _clear 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head#a = 0 2019-12-07T19:34:00.662+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fb500 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fb500 onodes shared_blobs 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fb500 allocated 0x[] released 0x[273000~31000] 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv release 0x273000~31000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 freelist release 0x273000~31000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 freelist _xor first_key 0x200000 last_key 0x280000 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 prepare 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fb500 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 io_done 2019-12-07T19:34:00.662+0000 7fe2caa6c700 10 osd.0 23 dequeue_op 0x558e83b0b080 finish 2019-12-07T19:34:00.662+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:00.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:00.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1279 < min_flush_size 524288 2019-12-07T19:34:00.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:00.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03900 had 1 2019-12-07T19:34:00.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1302 < min_flush_size 524288 2019-12-07T19:34:00.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1302 < min_flush_size 524288 2019-12-07T19:34:00.663+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x69287~516 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.663+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x69287 0x69287~516 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.663+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:00.663+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x39287 2019-12-07T19:34:00.663+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x287 2019-12-07T19:34:00.663+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:00.663+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x79d and padding block with 0x863 2019-12-07T19:34:00.663+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 50 ==== MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028c00 con 0x558e818fe400 2019-12-07T19:34:00.663+0000 7fe2e6d3a700 15 osd.0 23 enqueue_op 0x558e83b0a6e0 prio 196 cost 0 latency 0.000019 epoch 23 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.663+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 18 ==== MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84029600 con 0x558e818d6800 2019-12-07T19:34:00.663+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) 2019-12-07T19:34:00.663+0000 7fe2e753b700 15 osd.0 23 enqueue_op 0x558e83b0bb80 prio 196 cost 0 latency 0.000015 epoch 23 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.663+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) 2019-12-07T19:34:00.663+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 50 ==== MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e811f6600 con 0x558e818d7000 2019-12-07T19:34:00.663+0000 7fe2e7d3c700 15 osd.0 23 enqueue_op 0x558e83eddce0 prio 196 cost 0 latency 0.000017 epoch 23 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.663+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) 2019-12-07T19:34:00.663+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x6979d 2019-12-07T19:34:00.663+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:00.663+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:00.663+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:00.663+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:00.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000247340s (0.000000096s flush + 0.000247244s kv commit) 2019-12-07T19:34:00.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:00.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:00.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fb500> 2019-12-07T19:34:00.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:00.663+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 kv_submitted 2019-12-07T19:34:00.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fb500 2019-12-07T19:34:00.663+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 finishing 2019-12-07T19:34:00.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fb500 onodes 2019-12-07T19:34:00.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb500 done 2019-12-07T19:34:00.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:00.663+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fb500 [273000~31000] 2019-12-07T19:34:00.663+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release 0x273000~31000 2019-12-07T19:34:00.663+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:00.663+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) queued 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) pg 0x558e81075000 2019-12-07T19:34:00.663+0000 7fe2caa6c700 10 osd.0 23 dequeue_op 0x558e83b0a6e0 prio 196 cost 0 latency 0.000259 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] 2019-12-07T19:34:00.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.663+0000 7fe2caa6c700 10 osd.0 23 dequeue_op 0x558e83b0a6e0 finish 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) queued 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) pg 0x558e81075000 2019-12-07T19:34:00.663+0000 7fe2caa6c700 10 osd.0 23 dequeue_op 0x558e83b0bb80 prio 196 cost 0 latency 0.000291 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] 2019-12-07T19:34:00.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.663+0000 7fe2caa6c700 10 osd.0 23 dequeue_op 0x558e83b0bb80 finish 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) queued 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) pg 0x558e81075000 2019-12-07T19:34:00.663+0000 7fe2caa6c700 10 osd.0 23 dequeue_op 0x558e83eddce0 prio 196 cost 0 latency 0.000201 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] 2019-12-07T19:34:00.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.663+0000 7fe2caa6c700 10 osd.0 23 dequeue_op 0x558e83eddce0 finish 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.663+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.6 v2:172.21.15.109:6818/36937 22 ==== MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83fefe00 con 0x558e818ffc00 2019-12-07T19:34:00.663+0000 7fe2e7d3c700 15 osd.0 23 enqueue_op 0x558e83edde40 prio 196 cost 0 latency 0.000017 epoch 23 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.663+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) queued 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) pg 0x558e81075000 2019-12-07T19:34:00.663+0000 7fe2caa6c700 10 osd.0 23 dequeue_op 0x558e83edde40 prio 196 cost 0 latency 0.000059 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] 2019-12-07T19:34:00.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.663+0000 7fe2caa6c700 10 osd.0 23 dequeue_op 0x558e83edde40 finish 2019-12-07T19:34:00.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.664+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.665+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 93 ==== MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83fee200 con 0x558e818d7c00 2019-12-07T19:34:00.665+0000 7fe2e7d3c700 15 osd.0 23 enqueue_op 0x558e83b0af20 prio 196 cost 0 latency 0.000017 epoch 23 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.665+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) 2019-12-07T19:34:00.665+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:00.665+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) queued 2019-12-07T19:34:00.665+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:00.665+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2) prio 196 cost 0 e23) pg 0x558e81075000 2019-12-07T19:34:00.665+0000 7fe2caa6c700 10 osd.0 23 dequeue_op 0x558e83b0af20 prio 196 cost 0 latency 0.000059 MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] 2019-12-07T19:34:00.665+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 23/18 ECSubWriteReply(tid=16, last_complete=23'10, committed=1, applied=1)) v2 2019-12-07T19:34:00.665+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] try_finish_rmw: Op(1:4c9e444c:::smithi13140796-48:head v=0'0 tt=0'0 tid=16 reqid=client.4540.0:183 roll_forward_to=23'10 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:00.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:00.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:00.665+0000 7fe2caa6c700 10 osd.0 23 dequeue_op 0x558e83b0af20 finish 2019-12-07T19:34:00.665+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:00.681+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/890522420 conn(0x558e83d41800 msgr2=0x558e81846100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 98 2019-12-07T19:34:00.681+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/890522420 conn(0x558e83d41800 msgr2=0x558e81846100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:00.681+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/890522420 conn(0x558e83d41800 0x558e81846100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:00.681+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/890522420 conn(0x558e83d41800 0x558e81846100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:00.681+0000 7fe2d959f700 2 osd.0 23 ms_handle_reset con 0x558e83d41800 session 0x558e818e9a00 2019-12-07T19:34:00.687+0000 7fe2e4535700 10 osd.0 23 tick 2019-12-07T19:34:00.687+0000 7fe2e4535700 10 osd.0 23 do_waiters -- start 2019-12-07T19:34:00.687+0000 7fe2e4535700 10 osd.0 23 do_waiters -- finish 2019-12-07T19:34:00.687+0000 7fe2e4535700 20 osd.0 23 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:00.695+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 10 osd.0 23 tick_without_osd_lock 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fd60000/0x40000000/0x1680000000, data 0x277aae/0x29e000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 25 osd.0 23 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453695+0000 last_rx_front 2019-12-07T19:33:56.453704+0000 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 25 osd.0 23 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453559+0000 last_rx_front 2019-12-07T19:33:56.453627+0000 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 25 osd.0 23 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453575+0000 last_rx_front 2019-12-07T19:33:56.453596+0000 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 25 osd.0 23 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453839+0000 last_rx_front 2019-12-07T19:33:56.453807+0000 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 25 osd.0 23 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453763+0000 last_rx_front 2019-12-07T19:33:56.453773+0000 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 25 osd.0 23 heartbeat_check osd.6 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453751+0000 last_rx_front 2019-12-07T19:33:56.453819+0000 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 25 osd.0 23 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453789+0000 last_rx_front 2019-12-07T19:33:56.453733+0000 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 20 osd.0 23 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 20 osd.0 23 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 20 osd.0 23 scrub_load_below_threshold loadavg per cpu 0.065 < max 5 = yes 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 20 osd.0 23 sched_scrub load_is_low=1 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 10 osd.0 23 sched_scrub 1.2s0 scheduled at 2019-12-07T19:35:23.718605+0000 > 2019-12-07T19:34:00.727544+0000 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 20 osd.0 23 sched_scrub done 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 10 osd.0 23 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 20 osd.0 23 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:00.726+0000 7fe2e2ca2700 10 osd.0 23 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:00.745+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.795+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.846+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.896+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.946+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:00.996+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.046+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.072+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.7 172.21.15.109:0/36939 7 ==== osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.690383489s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83fef800 con 0x558e81749000 2019-12-07T19:34:01.072+0000 7fe2e6d3a700 20 osd.0 23 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690015972s,18446744073.690383489s]) 2019-12-07T19:34:01.072+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.527979159s delta_ub 0.019535644s) v5 -- 0x558e84029e00 con 0x558e81749000 2019-12-07T19:34:01.072+0000 7fe2e6d3a700 10 osd.0 23 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 21 -> 23 (as per caller) 2019-12-07T19:34:01.072+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.7 172.21.15.109:0/36939 7 ==== osd_ping(ping e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.508443515s delta_ub 18446744073.690383489s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83fef800 con 0x558e81748c00 2019-12-07T19:34:01.072+0000 7fe2e6d3a700 20 osd.0 23 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.689962988s,18446744073.690383489s]) 2019-12-07T19:34:01.072+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e23 up_from 13 ping_stamp 2019-12-07T19:34:01.073004+0000/24.508443515s send_stamp 24.528032143s delta_ub 0.019588628s) v5 -- 0x558e818a9e00 con 0x558e81748c00 2019-12-07T19:34:01.096+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.146+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.196+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.247+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.109:0/36937 conn(0x558e802a4c00 msgr2=0x558e80efb700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 61 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.109:0/36937 conn(0x558e802a4c00 msgr2=0x558e80efb700 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.109:0/36937 conn(0x558e802a4c00 0x558e80efb700 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] >> 172.21.15.109:0/36937 conn(0x558e802a4c00 0x558e80efb700 crc :-1 s=READY pgs=2 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:01.284+0000 7fe2d0a78700 20 osd.0 23 heartbeat_reset con 0x558e802a4c00 s 0x558e81807860 2019-12-07T19:34:01.284+0000 7fe2d0a78700 10 osd.0 23 heartbeat_reset closing (old) failed hb con 0x558e802a4c00 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.109:0/36937 conn(0x558e8182cc00 msgr2=0x558e81844000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 77 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.109:0/36937 conn(0x558e8182cc00 msgr2=0x558e81844000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.109:0/36937 conn(0x558e8182cc00 0x558e81844000 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] >> 172.21.15.109:0/36937 conn(0x558e8182cc00 0x558e81844000 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558e8181a800 msgr2=0x558e81817080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 88 2019-12-07T19:34:01.284+0000 7fe2d1a7a700 20 osd.0 23 heartbeat_reset con 0x558e8182cc00 s 0x558e818c21a0 2019-12-07T19:34:01.284+0000 7fe2d1a7a700 10 osd.0 23 heartbeat_reset closing (old) failed hb con 0x558e8182cc00 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558e8181a800 msgr2=0x558e81817080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558e8181a800 0x558e81817080 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558e8181a800 0x558e81817080 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:01.284+0000 7fe2d2a7c700 20 osd.0 23 heartbeat_reset con 0x558e8181a800 s 0x558e81807040 2019-12-07T19:34:01.284+0000 7fe2d2a7c700 10 osd.0 23 heartbeat_reset failed hb con 0x558e8181a800 for osd.6, reopening 2019-12-07T19:34:01.284+0000 7fe2d2a7c700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558e8181ac00 msgr2=0x558e81817600 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2019-12-07T19:34:01.284+0000 7fe2d2a7c700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558e8181ac00 0x558e81817600 crc :-1 s=READY pgs=6 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:01.284+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818ffc00 msgr2=0x558e818e7180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 96 2019-12-07T19:34:01.284+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818ffc00 msgr2=0x558e818e7180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2019-12-07T19:34:01.284+0000 7fe2d2a7c700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558e81748400 0x558e80efac00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:34:01.284+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818ffc00 0x558e818e7180 unknown :-1 s=READY pgs=16 cs=1 l=0 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:01.284+0000 7fe2d2a7c700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558e83ab4c00 0x558e84036000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rx=0 tx=0).connect 2019-12-07T19:34:01.284+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818ffc00 0x558e818e7180 unknown :-1 s=READY pgs=16 cs=1 l=0 rx=0 tx=0)._fault with nothing to send, going to standby 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558e81748400 msgr2=0x558e80efac00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.109:6822/36937 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558e81748400 0x558e80efac00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.200000 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558e83ab4c00 msgr2=0x558e84036000 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.109:6820/36937 2019-12-07T19:34:01.284+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558e83ab4c00 0x558e84036000 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.200000 2019-12-07T19:34:01.284+0000 7fe2d2a7c700 2 osd.0 23 ms_handle_refused con 0x558e81748400 session 0x558e81807040 2019-12-07T19:34:01.284+0000 7fe2d3a7e700 2 osd.0 23 ms_handle_refused con 0x558e83ab4c00 session 0x558e81807040 2019-12-07T19:34:01.284+0000 7fe2d2a7c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- osd_failure(failed immediate osd.6 [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] for 21sec e23 v23) v4 -- 0x558e83f2a240 con 0x558e80f39000 2019-12-07T19:34:01.284+0000 7fe2d3a7e700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- osd_failure(failed immediate osd.6 [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] for 21sec e23 v23) v4 -- 0x558e83f2a480 con 0x558e80f39000 2019-12-07T19:34:01.297+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.347+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.397+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.447+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.485+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558e81748400 msgr2=0x558e80efac00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.109:6822/36937 2019-12-07T19:34:01.485+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558e81748400 0x558e80efac00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.400000 2019-12-07T19:34:01.485+0000 7fe2d2a7c700 2 osd.0 23 ms_handle_refused con 0x558e81748400 session 0x558e81807040 2019-12-07T19:34:01.485+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558e83ab4c00 msgr2=0x558e84036000 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.109:6820/36937 2019-12-07T19:34:01.485+0000 7fe2e753b700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558e83ab4c00 0x558e84036000 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0)._fault waiting 0.400000 2019-12-07T19:34:01.485+0000 7fe2d2a7c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- osd_failure(failed immediate osd.6 [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] for 21sec e23 v23) v4 -- 0x558e83f2a6c0 con 0x558e80f39000 2019-12-07T19:34:01.485+0000 7fe2d3a7e700 2 osd.0 23 ms_handle_refused con 0x558e83ab4c00 session 0x558e81807040 2019-12-07T19:34:01.485+0000 7fe2d3a7e700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- osd_failure(failed immediate osd.6 [v2:172.21.15.109:6816/36937,v1:172.21.15.109:6817/36937] for 21sec e23 v23) v4 -- 0x558e83f2ab40 con 0x558e80f39000 2019-12-07T19:34:01.497+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.547+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.561+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:01.597+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:34:01.597+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:34:01.597+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:34:01.597+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:34:01.597+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:34:01.597+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:34:01.597+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:34:01.597+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:34:01.597+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:34:01.597+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:34:01.597+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:34:01.597+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:34:01.597+0000 7fe2dc28f700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:34:01.597+0000 7fe2dc28f700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258863 2019-12-07T19:34:01.597+0000 7fe2dc28f700 5 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 286312 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.644+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 51 ==== pg_query2(1.1s1 1.1s1 query(info 0'0 epoch_sent 24) e24/24) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8191bb80 con 0x558e818d7000 2019-12-07T19:34:01.644+0000 7fe2e7d3c700 15 osd.0 23 enqueue_peering_evt 1.1s1 epoch_sent: 24 epoch_requested: 24 MQuery 1.1s1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.644+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.1s1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.644+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 53 ==== pg_notify2(1.2s0 1.2s0 (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973692310s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e24/24) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e8122b100 con 0x558e818d7000 2019-12-07T19:34:01.644+0000 7fe2e7d3c700 15 osd.0 23 enqueue_peering_evt 1.2s0 epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 3(5) notify: (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973692310s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.644+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 3(5) notify: (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973692310s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) 2019-12-07T19:34:01.644+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.644+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 3(5) notify: (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973692310s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.644+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)> waiting <> waiting_peering {} 2019-12-07T19:34:01.644+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 3(5) notify: (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973692310s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) pg 0x558e81075000 2019-12-07T19:34:01.644+0000 7fe2c6a64700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 24, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 3(5) notify: (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973692310s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) 2019-12-07T19:34:01.644+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:01.645+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.645+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.1s1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.645+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.645+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.1s1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558e819e4000 2019-12-07T19:34:01.645+0000 7fe2cb26d700 20 osd.0 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 24, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.1s1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.645+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:01.646+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 52 ==== osd_map(24..24 src has 1..24) v4 ==== 964+0+0 (unknown 0 0 0) 0x558e83f2ab40 con 0x558e818d7000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 osd.0 23 OSD::ms_dispatch: osd_map(24..24 src has 1..24) v4 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 osd.0 23 do_waiters -- start 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 osd.0 23 do_waiters -- finish 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 osd.0 23 _dispatch 0x558e83f2ab40 osd_map(24..24 src has 1..24) v4 2019-12-07T19:34:01.646+0000 7fe2d4a80700 3 osd.0 23 handle_osd_map epochs [24,24], i have 23, src has [1,24] 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 osd.0 23 handle_osd_map got inc map for epoch 24 2019-12-07T19:34:01.646+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3de0e826:::osdmap.23:0# 0x0~0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1444 size 0x1444 (5188) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03ce0 blob([0xff000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d03c70 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03dc0 blob([!~1000,0x100000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,444]) SharedBlob(0x558e83d03d50 sbid 0x0)) need 0x1000~444 cache has 0x[1000~444] 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3de0e826:::osdmap.23:0# 0x0~1444 = 5188 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 osd.0 23 add_map_bl 23 5188 bytes 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 osd.0 23 got_full_map 24, nothing requested 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 osd.0 23 handle_osd_map pg_num_history pg_num_history(e24 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps purged_snaps {24={}} 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 osd.0 23 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e24 [1,24] lci=[13,24]) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e819a0900 seq 16 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:ce7b972f:::inc_osdmap.24:0# key 0x7f7fffffffffffffffce7b97'/!inc_osdmap.24!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:ce7b972f:::inc_osdmap.24:0# 0x0~398 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1111 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:ce7b972f:::inc_osdmap.24:0# 0x0~398 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~398 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xc68 on front/back, now 0x0~1000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0900 1 blobs 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x101000~1000/1000,1000,0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x101000~1000] 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af8fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af8f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af8fc0 blob([0x101000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8f50 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.blob(0x558e83af8fc0) get_ref 0x0~398 Blob(0x558e83af8fc0 blob([0x101000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8f50 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.blob(0x558e83af8fc0) get_ref init 0x1000, 1000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~398: 0x0~398 Blob(0x558e83af8fc0 blob([0x101000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x398) SharedBlob(0x558e83af8f50 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e83af8f68 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x398 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191b508) dirty_range mark inline shard dirty 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:ce7b972f:::inc_osdmap.24:0# 0x0~398 = 0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:32e0e826:::osdmap.24:0# key 0x7f7fffffffffffffff32e0e8'&!osdmap.24!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:32e0e826:::osdmap.24:0# 0x0~1428 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1112 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:32e0e826:::osdmap.24:0# 0x0~1428 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~428 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbd8 on front/back, now 0x0~1000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0900 2 blobs 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x102000~2000/1000,2000,0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x102000~2000] 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af8e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af8e70 blob([0x102000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f730 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.blob(0x558e83af8e70) get_ref 0x0~1000 Blob(0x558e83af8e70 blob([0x102000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f730 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.blob(0x558e83af8e70) get_ref init 0x1000, 1000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e83af8e70 blob([0x102000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d0f730 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e83d0f748 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0f030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0f030 blob([!~1000,0x103000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f0a0 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.blob(0x558e83d0f030) get_ref 0x1000~428 Blob(0x558e83d0f030 blob([!~1000,0x103000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f0a0 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.blob(0x558e83d0f030) get_ref init 0x2000, 1000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~428: 0x1000~428 Blob(0x558e83d0f030 blob([!~1000,0x103000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,428]) SharedBlob(0x558e83d0f0a0 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e83d0f0b8 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1428 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83f29a08) dirty_range mark inline shard dirty 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:32e0e826:::osdmap.24:0# 0x0~1428 = 0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0900 0 blobs 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0900 0 blobs 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0900 cost 23561 (4 ios * 4000 + 7561 bytes) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0900 onodes 0x558e80251b80,0x558e817b9180,0x558e8191b400,0x558e83f29900 shared_blobs 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191b508) update #-1:ce7b972f:::inc_osdmap.24:0# 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191b508) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:ce7b972f:::inc_osdmap.24:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83f29a08) update #-1:32e0e826:::osdmap.24:0# 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83f29a08) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:32e0e826:::osdmap.24:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0900 allocated 0x[101000~3000] released 0x[] 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 freelist allocate 0x101000~3000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 prepare 2019-12-07T19:34:01.646+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e819a0900 2019-12-07T19:34:01.647+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 20 ==== pg_notify2(1.2s0 1.2s0 (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e24/24) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e8122b800 con 0x558e818d6800 2019-12-07T19:34:01.647+0000 7fe2e753b700 15 osd.0 23 enqueue_peering_evt 1.2s0 epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.647+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) 2019-12-07T19:34:01.647+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {24=0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)>} 2019-12-07T19:34:01.647+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.647+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)> waiting <> waiting_peering {24=0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)>} 2019-12-07T19:34:01.647+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) pg 0x558e81075000 2019-12-07T19:34:01.647+0000 7fe2caa6c700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 24, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) 2019-12-07T19:34:01.647+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:01.647+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 51 ==== pg_notify2(1.2s0 1.2s0 (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973540341s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e24/24) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818fe400 2019-12-07T19:34:01.647+0000 7fe2e6d3a700 15 osd.0 23 enqueue_peering_evt 1.2s0 epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973540341s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.647+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973540341s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) 2019-12-07T19:34:01.647+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 19 ==== osd_map(24..24 src has 1..24) v4 ==== 964+0+0 (unknown 0 0 0) 0x558e8123b440 con 0x558e818d6800 2019-12-07T19:34:01.647+0000 7fe2d4a80700 20 osd.0 23 OSD::ms_dispatch: osd_map(24..24 src has 1..24) v4 2019-12-07T19:34:01.647+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 52 ==== pg_query2(1.bs3 1.bs3 query(info 0'0 epoch_sent 24) e24/24) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8402e780 con 0x558e818fe400 2019-12-07T19:34:01.647+0000 7fe2d4a80700 10 osd.0 23 do_waiters -- start 2019-12-07T19:34:01.647+0000 7fe2d4a80700 10 osd.0 23 do_waiters -- finish 2019-12-07T19:34:01.647+0000 7fe2d4a80700 20 osd.0 23 _dispatch 0x558e8123b440 osd_map(24..24 src has 1..24) v4 2019-12-07T19:34:01.647+0000 7fe2e6d3a700 15 osd.0 23 enqueue_peering_evt 1.bs3 epoch_sent: 24 epoch_requested: 24 MQuery 1.bs3 from 4(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.647+0000 7fe2d4a80700 3 osd.0 23 handle_osd_map epochs [24,24], i have 24, src has [1,24] 2019-12-07T19:34:01.647+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.bs3 from 4(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.647+0000 7fe2d4a80700 10 osd.0 23 no new maps here, dropping 2019-12-07T19:34:01.647+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 53 ==== osd_map(24..24 src has 1..24) v4 ==== 964+0+0 (unknown 0 0 0) 0x558e8123afc0 con 0x558e818fe400 2019-12-07T19:34:01.647+0000 7fe2d4a80700 20 osd.0 23 OSD::ms_dispatch: osd_map(24..24 src has 1..24) v4 2019-12-07T19:34:01.647+0000 7fe2d4a80700 10 osd.0 23 do_waiters -- start 2019-12-07T19:34:01.647+0000 7fe2d4a80700 10 osd.0 23 do_waiters -- finish 2019-12-07T19:34:01.647+0000 7fe2d4a80700 20 osd.0 23 _dispatch 0x558e8123afc0 osd_map(24..24 src has 1..24) v4 2019-12-07T19:34:01.647+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.647+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.bs3 from 4(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.647+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.647+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.bs3 from 4(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558e81928000 2019-12-07T19:34:01.647+0000 7fe2c6263700 20 osd.0 op_wq _add_slot_waiter 1.bs3 peering, item epoch is 24, will wait on OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.bs3 from 4(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.647+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:01.647+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {24=0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)>} 2019-12-07T19:34:01.647+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973540341s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.647+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)> waiting <> waiting_peering {24=0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)>} 2019-12-07T19:34:01.647+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973540341s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) pg 0x558e81075000 2019-12-07T19:34:01.647+0000 7fe2c6a64700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 24, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973540341s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) 2019-12-07T19:34:01.647+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:01.647+0000 7fe2d4a80700 3 osd.0 23 handle_osd_map epochs [24,24], i have 24, src has [1,24] 2019-12-07T19:34:01.647+0000 7fe2d4a80700 10 osd.0 23 no new maps here, dropping 2019-12-07T19:34:01.647+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 aio_wait 2019-12-07T19:34:01.647+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0900 2019-12-07T19:34:01.647+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 io_done 2019-12-07T19:34:01.647+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:01.647+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:01.647+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:01.647+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:01.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:01.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:01.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8191b400 had 1 2019-12-07T19:34:01.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83f29900 had 1 2019-12-07T19:34:01.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:01.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:01.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:01.648+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x6979d~226c to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.648+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x6979d 0x6979d~226c to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.648+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.648+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3979d 2019-12-07T19:34:01.648+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x79d 2019-12-07T19:34:01.648+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:01.648+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xa09 and padding block with 0x5f7 2019-12-07T19:34:01.648+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.649+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 54 ==== pg_query2(1.fs1 1.fs1 query(info 0'0 epoch_sent 24) e24/24) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8191bb80 con 0x558e818d7000 2019-12-07T19:34:01.649+0000 7fe2e7d3c700 15 osd.0 23 enqueue_peering_evt 1.fs1 epoch_sent: 24 epoch_requested: 24 MQuery 1.fs1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.649+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.fs1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.649+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 94 ==== pg_query2(1.9s3 1.9s3 query(info 0'0 epoch_sent 24) e24/24) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8191bb80 con 0x558e818d7c00 2019-12-07T19:34:01.649+0000 7fe2e7d3c700 15 osd.0 23 enqueue_peering_evt 1.9s3 epoch_sent: 24 epoch_requested: 24 MQuery 1.9s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.649+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.9s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.649+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 95 ==== pg_notify2(1.2s0 1.2s0 (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.970353093s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e24/24) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e8122b100 con 0x558e818d7c00 2019-12-07T19:34:01.649+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.649+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.fs1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.649+0000 7fe2e7d3c700 15 osd.0 23 enqueue_peering_evt 1.2s0 epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 1(2) notify: (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.970353093s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.649+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.649+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 1(2) notify: (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.970353093s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) 2019-12-07T19:34:01.649+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.fs1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558e81a9b000 2019-12-07T19:34:01.649+0000 7fe2c425f700 20 osd.0 op_wq _add_slot_waiter 1.fs1 peering, item epoch is 24, will wait on OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.fs1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.649+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:01.649+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 96 ==== pg_query2(1.7s3 1.7s3 query(info 0'0 epoch_sent 24) e24/24) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8191bb80 con 0x558e818d7c00 2019-12-07T19:34:01.649+0000 7fe2e7d3c700 15 osd.0 23 enqueue_peering_evt 1.7s3 epoch_sent: 24 epoch_requested: 24 MQuery 1.7s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.649+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.7s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 98 ==== pg_query2(1.as3 1.as3 query(info 0'0 epoch_sent 24) e24/24) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8191bb80 con 0x558e818d7c00 2019-12-07T19:34:01.650+0000 7fe2e7d3c700 15 osd.0 23 enqueue_peering_evt 1.as3 epoch_sent: 24 epoch_requested: 24 MQuery 1.as3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.650+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.as3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {24=0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973540341s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)>} 2019-12-07T19:34:01.650+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 1(2) notify: (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.970353093s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.650+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)> waiting <> waiting_peering {24=0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973540341s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)>} 2019-12-07T19:34:01.650+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.650+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 1(2) notify: (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.970353093s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) pg 0x558e81075000 2019-12-07T19:34:01.650+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.650+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.7s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.650+0000 7fe2caa6c700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 24, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 1(2) notify: (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.970353093s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.9s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.650+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.650+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.650+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.650+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.7s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558e8122e000 2019-12-07T19:34:01.650+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.9s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558e8192b000 2019-12-07T19:34:01.650+0000 7fe2c8267700 20 osd.0 op_wq _add_slot_waiter 1.7s3 peering, item epoch is 24, will wait on OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.7s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2c7265700 20 osd.0 op_wq _add_slot_waiter 1.9s3 peering, item epoch is 24, will wait on OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.9s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:01.650+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 97 ==== osd_map(24..24 src has 1..24) v4 ==== 964+0+0 (unknown 0 0 0) 0x558e83f2a6c0 con 0x558e818d7c00 2019-12-07T19:34:01.650+0000 7fe2d4a80700 20 osd.0 23 OSD::ms_dispatch: osd_map(24..24 src has 1..24) v4 2019-12-07T19:34:01.650+0000 7fe2d4a80700 10 osd.0 23 do_waiters -- start 2019-12-07T19:34:01.650+0000 7fe2d4a80700 10 osd.0 23 do_waiters -- finish 2019-12-07T19:34:01.650+0000 7fe2d4a80700 20 osd.0 23 _dispatch 0x558e83f2a6c0 osd_map(24..24 src has 1..24) v4 2019-12-07T19:34:01.650+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.as3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.650+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.650+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.as3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558e8193b000 2019-12-07T19:34:01.650+0000 7fe2caa6c700 20 osd.0 op_wq _add_slot_waiter 1.as3 peering, item epoch is 24, will wait on OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.as3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:01.650+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:01.650+0000 7fe2d4a80700 3 osd.0 23 handle_osd_map epochs [24,24], i have 24, src has [1,24] 2019-12-07T19:34:01.650+0000 7fe2d4a80700 10 osd.0 23 no new maps here, dropping 2019-12-07T19:34:01.650+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x6ba09 2019-12-07T19:34:01.650+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:01.650+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:01.650+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:01.650+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:01.650+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.002658501s (0.000005585s flush + 0.002652916s kv commit) 2019-12-07T19:34:01.650+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:01.650+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:01.650+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a0900> 2019-12-07T19:34:01.650+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:01.650+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 kv_submitted 2019-12-07T19:34:01.650+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0900 2019-12-07T19:34:01.650+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e819a0900 osr 0x558e8107a500 2019-12-07T19:34:01.650+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) _discard drop 13 0x1b000~1000 2019-12-07T19:34:01.650+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) prepare_write seq 14 0x1b000~1000 crc a6a9dc77 2019-12-07T19:34:01.650+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) _discard drop 13 0x2000~1000 2019-12-07T19:34:01.650+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) prepare_write seq 14 0x2000~1000 crc 1ff68305 2019-12-07T19:34:01.650+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:01.650+0000 7fe2d7285700 10 osd.0 23 _committed_osd_maps 24..24 2019-12-07T19:34:01.650+0000 7fe2d7285700 10 osd.0 23 advance to epoch 24 (<= last 24 <= newest_map 24) 2019-12-07T19:34:01.650+0000 7fe2d7285700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] mark_down_addrs [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] -- 0x558e818ffc00 2019-12-07T19:34:01.650+0000 7fe2d7285700 1 --2- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] >> [v2:172.21.15.109:6818/36937,v1:172.21.15.109:6819/36937] conn(0x558e818ffc00 0x558e818e7180 unknown :-1 s=STANDBY pgs=16 cs=1 l=0 rx=0 tx=0).stop 2019-12-07T19:34:01.650+0000 7fe2d7285700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558e81748400 msgr2=0x558e80efac00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2019-12-07T19:34:01.650+0000 7fe2d7285700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6822/36937,v1:172.21.15.109:6823/36937] conn(0x558e81748400 0x558e80efac00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:01.650+0000 7fe2d7285700 1 -- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558e83ab4c00 msgr2=0x558e84036000 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2019-12-07T19:34:01.650+0000 7fe2d7285700 1 --2- 172.21.15.131:0/37079 >> [v2:172.21.15.109:6820/36937,v1:172.21.15.109:6821/36937] conn(0x558e83ab4c00 0x558e84036000 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:01.650+0000 7fe2d7285700 7 osd.0 24 consume_map version 24 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 24 identify_splits_and_merges 1.9s3 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 24 identify_splits_and_merges 1.1s1 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 24 identify_splits_and_merges 1.as3 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 24 identify_splits_and_merges 1.2s0 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 24 identify_splits_and_merges 1.bs3 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 24 identify_splits_and_merges 1.4s1 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 24 identify_splits_and_merges 1.ds5 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 24 identify_splits_and_merges 1.es4 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 24 identify_splits_and_merges 1.7s3 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 24 identify_splits_and_merges 1.fs1 e23 to e24 pg_nums {18=16} 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 24 prune_pg_created 2019-12-07T19:34:01.650+0000 7fe2d7285700 10 osd.0:0.consume_map 24 (was 23) 2019-12-07T19:34:01.650+0000 7fe2d7285700 10 osd.0:1.consume_map 24 (was 23) 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 pending_peering first epoch 24 <= 24, requeueing 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.9s3 to_process <> waiting <> waiting_peering {24=} 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 pending_peering first epoch 24 <= 24, requeueing 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {24=} 2019-12-07T19:34:01.650+0000 7fe2d7285700 10 osd.0:2.consume_map 24 (was 23) 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 pending_peering first epoch 24 <= 24, requeueing 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.as3 to_process <> waiting <> waiting_peering {24=} 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 pending_peering first epoch 24 <= 24, requeueing 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.2s0 to_process <> waiting <> waiting_peering {24=0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973540341s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 1(2) notify: (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.970353093s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)>} 2019-12-07T19:34:01.650+0000 7fe2d7285700 10 osd.0:3.consume_map 24 (was 23) 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 pending_peering first epoch 24 <= 24, requeueing 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.bs3 to_process <> waiting <> waiting_peering {24=} 2019-12-07T19:34:01.650+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.650+0000 7fe2d7285700 10 osd.0:4.consume_map 24 (was 23) 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:01.650+0000 7fe2d7285700 10 osd.0:5.consume_map 24 (was 23) 2019-12-07T19:34:01.650+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.1s1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:01.650+0000 7fe2d7285700 10 osd.0:6.consume_map 24 (was 23) 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:01.650+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.650+0000 7fe2d7285700 10 osd.0:7.consume_map 24 (was 23) 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:01.650+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.1s1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558e819e4000 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 pending_peering first epoch 24 <= 24, requeueing 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.7s3 to_process <> waiting <> waiting_peering {24=} 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 pending_peering first epoch 24 <= 24, requeueing 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.fs1 to_process <> waiting <> waiting_peering {24=} 2019-12-07T19:34:01.650+0000 7fe2d7285700 10 osd.0 24 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:01.650+0000 7fe2d7285700 15 osd.0 24 enqueue_peering_evt 1.9s3 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2cb26d700 10 osd.0 pg_epoch: 23 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.650+0000 7fe2d7285700 15 osd.0 24 enqueue_peering_evt 1.1s1 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.650+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.1s1 23 -> 24 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2d7285700 15 osd.0 24 enqueue_peering_evt 1.as3 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2d7285700 15 osd.0 24 enqueue_peering_evt 1.2s0 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2cb26d700 10 osd.0 pg_epoch: 23 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] handle_advance_map [3,0,2147483647,4,5,1]/[3,0,2147483647,4,5,1] -- 3/3 2019-12-07T19:34:01.650+0000 7fe2d7285700 15 osd.0 24 enqueue_peering_evt 1.bs3 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2d7285700 15 osd.0 24 enqueue_peering_evt 1.4s1 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.650+0000 7fe2d7285700 15 osd.0 24 enqueue_peering_evt 1.ds5 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] new interval newup [3,0,2147483647,4,5,1] newacting [3,0,2147483647,4,5,1] 2019-12-07T19:34:01.650+0000 7fe2d7285700 15 osd.0 24 enqueue_peering_evt 1.es4 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.650+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:01.650+0000 7fe2d7285700 15 osd.0 24 enqueue_peering_evt 1.7s3 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.650+0000 7fe2d7285700 15 osd.0 24 enqueue_peering_evt 1.fs1 epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.650+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558e81938000 2019-12-07T19:34:01.650+0000 7fe2d7285700 7 osd.0 24 activate_map version 24 2019-12-07T19:34:01.650+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:01.651570+0000 15 0.000179 2019-12-07T19:34:01.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 3(5) notify: (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973692310s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.650+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:01.651582+0000 0 0.000000 2019-12-07T19:34:01.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)> waiting <> waiting_peering {} 2019-12-07T19:34:01.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 3(5) notify: (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973692310s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) pg 0x558e81075000 2019-12-07T19:34:01.650+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:01.651591+0000 0 0.000000 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 23 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 20 osd.0:4.update_pg_epoch 1.4s1 23 -> 24 2019-12-07T19:34:01.650+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 23 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:01.650+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=18 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.650+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.650+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.2s0 23 -> 24 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 24 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.650+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=24 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 24 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.650+0000 7fe2c6a64700 10 osd.0 pg_epoch: 23 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] handle_advance_map [0,4,1,2147483647,5,3]/[0,4,1,2147483647,5,3] -- 0/0 2019-12-07T19:34:01.650+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=24 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:34:01.650+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=24 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:01.650+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] new interval newup [0,4,1,2147483647,5,3] newacting [0,4,1,2147483647,5,3] 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 24 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.650+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=24 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] new interval newup [3,0,2147483647,4,5,1] newacting [3,0,2147483647,4,5,1] 2019-12-07T19:34:01.650+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] state: Active advmap interval change, fast return 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 24 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.650+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=24 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 24 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.650+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 20 osd.0 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.650+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,6,4,5,1]p3(0) r=1 lpr=24 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.650+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] new interval newup [0,4,1,2147483647,5,3] newacting [0,4,1,2147483647,5,3] 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 24 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.650+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 24 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.650+0000 7fe2cb26d700 15 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 24 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:01.650+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active+clean ps=[2~1]] exit Started/Primary/Active/Clean 2019-12-07T19:34:01.651664+0000 11 0.000534 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 24 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:01.650+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:01.650+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] exit Started/Primary/Active 2019-12-07T19:34:01.651677+0000 0 0.000000 2019-12-07T19:34:01.650+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.650+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(18-23 up [3,0,6,4,5,1](3) acting [3,0,6,4,5,1](3)) up_thru 18 up_from 13 last_epoch_clean 19 interval(18-23 up [3,0,6,4,5,1](3) acting [3,0,6,4,5,1](3) maybe_went_rw) : primary up 13-18 includes interval 2019-12-07T19:34:01.651+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] exit Started/Primary 2019-12-07T19:34:01.651697+0000 0 0.000000 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] noting past ([18,23] all_participants=0(1),1(5),3(0),4(3),5(4),6(2) intervals=([18,23] acting 0(1),1(5),3(0),4(3),5(4),6(2))) 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:01.651+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:01.651+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:01.651726+0000 0 0.000000 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'3 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d1570] 2019-12-07T19:34:01.651+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'3 active pruub 36.071368443s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.071368443s (mnow 25.106425972s + 10.964942471s) 2019-12-07T19:34:01.651+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=18 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'3 active pruub 36.071368443s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=24 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'3 active pruub 36.071368443s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=24 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:01.651+0000 7fe2cb26d700 1 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'3 active pruub 36.071368443s@ mbc={} ps=[2~1]] start_peering_interval up [3,0,6,4,5,1] -> [3,0,2147483647,4,5,1], acting [3,0,6,4,5,1] -> [3,0,2147483647,4,5,1], acting_primary 3(0) -> 3, up_primary 3(0) -> 3, role 1 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=24 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:01.651+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=24 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] new interval newup [0,4,1,2147483647,5,3] newacting [0,4,1,2147483647,5,3] 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown pruub 36.071368443s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=24 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown pruub 36.071368443s@ mbc={} ps=[2~1]] release_backoffs [1:80000000::::0,1:90000000::::head) 2019-12-07T19:34:01.651+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,6,5,3]p0(0) r=0 lpr=24 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 24 clear_ready_to_merge 1.2s0 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown pruub 36.071368443s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.651+0000 7fe2c6a64700 15 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown pruub 36.071368443s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown pruub 36.071368443s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown pruub 36.071368443s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown pruub 36.071368443s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(18-23 up [0,4,1,6,5,3](0) acting [0,4,1,6,5,3](0)) up_thru 18 up_from 13 last_epoch_clean 19 interval(18-23 up [0,4,1,6,5,3](0) acting [0,4,1,6,5,3](0) maybe_went_rw) : primary up 13-18 includes interval 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown pruub 36.071368443s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:01.651+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.651+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.651+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] noting past ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) 2019-12-07T19:34:01.651+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558e819e7000 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown pruub 36.071368443s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:01.651+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown pruub 36.071368443s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.651+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown pruub 36.071368443s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 21'9 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d0af0,0x558e811d01c0,0x558e811d18f0,0x558e811d1570] 2019-12-07T19:34:01.651+0000 7fe2c9269700 10 osd.0 pg_epoch: 23 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.651+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.ds5 23 -> 24 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown pruub 36.071368443s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 21'9 active pruub 36.073596236s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.073596236s (mnow 25.106553380s + 10.967042856s) 2019-12-07T19:34:01.651+0000 7fe2c9269700 10 osd.0 pg_epoch: 23 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.651+0000 7fe2c9269700 10 osd.0 pg_epoch: 24 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.651+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 21'9 active pruub 36.073596236s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:01.651+0000 7fe2c9269700 10 osd.0 pg_epoch: 24 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.651+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 21'9 active pruub 36.073596236s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:01.651+0000 7fe2c9269700 10 osd.0 pg_epoch: 24 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.651+0000 7fe2c9269700 20 osd.0 pg_epoch: 24 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 21'9 active pruub 36.073596236s@ mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:14.713285+0000, must = 0 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.651+0000 7fe2c9269700 20 osd.0 pg_epoch: 24 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.651+0000 7fe2c9269700 20 osd.0 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.651+0000 7fe2c6a64700 1 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 21'9 active pruub 36.073596236s@ mbc={} ps=[2~1]] start_peering_interval up [0,4,1,6,5,3] -> [0,4,1,2147483647,5,3], acting [0,4,1,6,5,3] -> [0,4,1,2147483647,5,3], acting_primary 0(0) -> 0, up_primary 0(0) -> 0, role 0 -> 0, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964942471s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.651+0000 7fe2c9269700 10 osd.0 pg_epoch: 24 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 21'9 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:01.651+0000 7fe2c9269700 10 osd.0 pg_epoch: 24 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.651+0000 7fe2c9269700 20 osd.0 pg_epoch: 24 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.651+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:01.651+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964764587s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:01.651917+0000 1 0.000330 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:01.651+0000 7fe2c9269700 10 osd.0 pg_epoch: 24 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.651+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:01.651+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964764587s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:01.651+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:01.651+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.651+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964764587s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:01.651+0000 7fe2d4a80700 2 osd.0 24 ms_handle_reset con 0x558e818ffc00 session 0x558e818c3860 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.651+0000 7fe2cb26d700 1 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964764587s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:01.651+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964764587s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:01.651947+0000 0 0.000000 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:01.651+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964764587s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964764587s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.651+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:01.651+0000 7fe2c6a64700 15 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] publish_stats_to_osd 23:33 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964764587s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MQuery 1.1s1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:01.651+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 24 send_incremental_map 23 -> 24 to 0x558e818d7000 v2:172.21.15.131:6812/37076 2019-12-07T19:34:01.651+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:01.651+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:ce7b972f:::inc_osdmap.24:0# 0x0~0 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~398 size 0x398 (920) 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8fc0 blob([0x101000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x398) SharedBlob(0x558e83af8f50 sbid 0x0)) need 0x0~398 cache has 0x[0~398] 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:ce7b972f:::inc_osdmap.24:0# 0x0~398 = 920 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] [0,4,1,6,5,3] -> [0,4,1,2147483647,5,3], replicas changed 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 24 add_map_inc_bl 24 920 bytes 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.651+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(24..24 src has 1..24) v4 -- 0x558e80238fc0 con 0x558e818d7000 2019-12-07T19:34:01.651+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 24 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 23 -> 24 (shared) 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.651+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.1s0 1.1s0 (query:24 sent:24 1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964764587s) 1->0 ([18,23] all_participants=0(1),1(5),3(0),4(3),5(4),6(2) intervals=([18,23] acting 0(1),1(5),3(0),4(3),5(4),6(2)))) e24/24) v1 -- 0x558e83ebb500 con 0x558e818d7000 2019-12-07T19:34:01.651+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.651+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.651+0000 7fe2c6a64700 20 osd.0 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.651+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.1s0 1.1s0 (query:24 sent:24 1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) 1->0 ([18,23] all_participants=0(1),1(5),3(0),4(3),5(4),6(2) intervals=([18,23] acting 0(1),1(5),3(0),4(3),5(4),6(2)))) e24/24) v1 -- 0x558e83ebc300 con 0x558e818d7000 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e8106c000 seq 12 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,6 -> 0,1,3,4,5 2019-12-07T19:34:01.651+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:01.651+0000 7fe2c6a64700 20 osd.0 24 need_heartbeat_peer_update 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:01.651+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:01.652104+0000 1 0.000423 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c000 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c000 onodes shared_blobs 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c000 allocated 0x[] released 0x[] 2019-12-07T19:34:01.651+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 prepare 2019-12-07T19:34:01.651+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c000 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 io_done 2019-12-07T19:34:01.651+0000 7fe2c6a64700 1 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] state: transitioning to Primary 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.651+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:01.652137+0000 0 0.000000 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.9s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.9s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558e8192b000 2019-12-07T19:34:01.651+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] enter Started/Primary 2019-12-07T19:34:01.651+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown pruub 36.073596236s@ mbc={} ps=[2~1]] enter Started/Primary/Peering 2019-12-07T19:34:01.651+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering pruub 36.073596236s@ mbc={} ps=[2~1]] enter Started/Primary/Peering/GetInfo 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 23 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.9s3 23 -> 24 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 23 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map [1,4,5,0,7,2147483647]/[1,4,5,0,7,2147483647] -- 1/1 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering pruub 36.073596236s@ mbc={} ps=[2~1]] build_prior all_probe 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering pruub 36.073596236s@ mbc={} ps=[2~1]] build_prior maybe_rw interval:18, acting: 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,2147483647] newacting [1,4,5,0,7,2147483647] 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering pruub 36.073596236s@ mbc={} ps=[2~1]] build_prior prior osd.6 is down 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering pruub 36.073596236s@ mbc={} ps=[2~1]] build_prior final: probe 0(0),1(2),3(5),4(1),5(4) down 6 blocked_by {} 2019-12-07T19:34:01.651+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:01.652207+0000 15 0.001119 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering pruub 36.073596236s@ mbc={} ps=[2~1]] up_thru 18 < same_since 24, must notify monitor 2019-12-07T19:34:01.651+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:01.652218+0000 0 0.000000 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering pruub 36.073596236s@6 mbc={} ps=[2~1]] state: querying info from osd.1(2) 2019-12-07T19:34:01.651+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:01.652226+0000 0 0.000000 2019-12-07T19:34:01.651+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering pruub 36.073596236s@6 mbc={} ps=[2~1]] state: querying info from osd.3(5) 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering pruub 36.073596236s@6 mbc={} ps=[2~1]] state: querying info from osd.4(1) 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering pruub 36.073596236s@6 mbc={} ps=[2~1]] state: querying info from osd.5(4) 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering pruub 36.073596236s@6 mbc={} ps=[2~1]] check_prior_readable_down_osds prior_readable_down_osds osd.6 is dead as of epoch 24 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,2147483647] newacting [1,4,5,0,7,2147483647] 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.bs3 from 4(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.bs3 from 4(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558e81928000 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:01.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,6]p1(0) r=3 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.651+0000 7fe2cb26d700 15 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 23 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs3 23 -> 24 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 23 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] handle_advance_map [4,2147483647,2,0,1,5]/[4,2147483647,2,0,1,5] -- 4/4 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] new interval newup [4,2147483647,2,0,1,5] newacting [4,2147483647,2,0,1,5] 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:01.651+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.651+0000 7fe2cc26f700 20 osd.0 24 agent_entry empty queue 2019-12-07T19:34:01.651+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.651+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.651+0000 7fe2ca26b700 5 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:01.652347+0000 15 0.000170 2019-12-07T19:34:01.651+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558e81a98000 2019-12-07T19:34:01.651+0000 7fe2ca26b700 5 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:01.652364+0000 0 0.000000 2019-12-07T19:34:01.651+0000 7fe2ca26b700 5 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:01.652373+0000 0 0.000000 2019-12-07T19:34:01.651+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.651+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.fs1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.651+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.651+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.fs1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558e81a9b000 2019-12-07T19:34:01.651+0000 7fe2ca26b700 5 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=18 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=24 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 23 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.651+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.fs1 23 -> 24 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=24 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 23 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map [3,0,5,4,1,2147483647]/[3,0,5,4,1,2147483647] -- 3/3 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=24 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=24 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] new interval newup [4,2147483647,2,0,1,5] newacting [4,2147483647,2,0,1,5] 2019-12-07T19:34:01.651+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] new interval newup [3,0,5,4,1,2147483647] newacting [3,0,5,4,1,2147483647] 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=24 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,6,2,0,1,5]p4(0) r=3 lpr=24 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.651+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:01.652433+0000 15 0.000156 2019-12-07T19:34:01.651+0000 7fe2ca26b700 15 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:01.651+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:01.652441+0000 0 0.000000 2019-12-07T19:34:01.651+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:01.652449+0000 0 0.000000 2019-12-07T19:34:01.651+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(18-23 up [4,6,2,0,1,5](4) acting [4,6,2,0,1,5](4)) up_thru 18 up_from 13 last_epoch_clean 19 interval(18-23 up [4,6,2,0,1,5](4) acting [4,6,2,0,1,5](4) maybe_went_rw) : primary up 13-18 includes interval 2019-12-07T19:34:01.651+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] noting past ([18,23] all_participants=0(3),1(4),2(2),4(0),5(5),6(1) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:01.651+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] new interval newup [3,0,5,4,1,2147483647] newacting [3,0,5,4,1,2147483647] 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d0af0] 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'12 active pruub 36.069286165s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.069286165s (mnow 25.107190761s + 10.962095404s) 2019-12-07T19:34:01.651+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,6]p3(0) r=1 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'12 active pruub 36.069286165s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:01.651+0000 7fe2c425f700 15 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'12 active pruub 36.069286165s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:01.651+0000 7fe2ca26b700 1 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'12 active pruub 36.069286165s@ mbc={} ps=[2~1]] start_peering_interval up [4,6,2,0,1,5] -> [4,2147483647,2,0,1,5], acting [4,6,2,0,1,5] -> [4,2147483647,2,0,1,5], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role 3 -> 3, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(18-23 up [3,0,5,4,1,6](3) acting [3,0,5,4,1,6](3)) up_thru 18 up_from 13 last_epoch_clean 19 interval(18-23 up [3,0,5,4,1,6](3) acting [3,0,5,4,1,6](3) maybe_went_rw) : primary up 13-18 includes interval 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown pruub 36.069286165s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown pruub 36.069286165s@ mbc={} ps=[2~1]] release_backoffs [1:d0000000::::0,1:e0000000::::head) 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] noting past ([18,23] all_participants=0(1),1(4),3(0),4(3),5(2),6(5) intervals=([18,23] acting 0(1),1(4),3(0),4(3),5(2),6(5))) 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown pruub 36.069286165s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.651+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown pruub 36.069286165s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.651+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown pruub 36.069286165s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown pruub 36.069286165s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d1570] 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown pruub 36.069286165s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active pruub 36.071369293s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.071369293s (mnow 25.107263959s + 10.964105334s) 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown pruub 36.069286165s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active pruub 36.071369293s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown pruub 36.069286165s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active pruub 36.071369293s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown pruub 36.069286165s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown pruub 36.069286165s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:01.651+0000 7fe2c425f700 1 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active pruub 36.071369293s@ mbc={} ps=[2~1]] start_peering_interval up [3,0,5,4,1,6] -> [3,0,5,4,1,2147483647], acting [3,0,5,4,1,6] -> [3,0,5,4,1,2147483647], acting_primary 3(0) -> 3, up_primary 3(0) -> 3, role 1 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown pruub 36.069286165s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.071369293s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.651+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.071369293s@ mbc={} ps=[2~1]] release_backoffs [1:f0000000::::0,MAX) 2019-12-07T19:34:01.651+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs3_head options 2019-12-07T19:34:01.651+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:01.651+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.071369293s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.071369293s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.071369293s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:01.651+0000 7fe2ca26b700 20 osd.0 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.071369293s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:01.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.071369293s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:01.651+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.962095404s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.651+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.071369293s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:01.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x6ba09~50c to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.651+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x6ba09 0x6ba09~50c to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.651+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.071369293s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:01.651+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3ba09 2019-12-07T19:34:01.651+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xa09 2019-12-07T19:34:01.651+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:01.651+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xf15 and padding block with 0xeb 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.071369293s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.651+0000 7fe2ca26b700 5 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961940673s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:01.652667+0000 1 0.000298 2019-12-07T19:34:01.651+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.071369293s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:01.651+0000 7fe2ca26b700 5 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961940673s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:01.651+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.071369293s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:01.651+0000 7fe2ca26b700 5 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961940673s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:01.651+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.652+0000 7fe2ca26b700 1 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961940673s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:01.652+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.652+0000 7fe2ca26b700 5 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961940673s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:01.652700+0000 0 0.000000 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.652+0000 7fe2ca26b700 5 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961940673s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.652+0000 7fe2ca26b700 20 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961940673s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x6bf15 2019-12-07T19:34:01.652+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964105334s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:01.652+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961940673s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MQuery 1.bs3 from 4(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000115638s (0.000000097s flush + 0.000115541s kv commit) 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:01.652+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963942827s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:01.652739+0000 1 0.000285 2019-12-07T19:34:01.652+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8106c000> 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:01.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 kv_submitted 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c000 2019-12-07T19:34:01.652+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963942827s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:01.652+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:01.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 finishing 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c000 onodes 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c000 done 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:01.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c000 [] 2019-12-07T19:34:01.652+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:01.652+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963942827s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:01.651+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering pruub 36.073596236s@ mbc={} ps=[2~1]] empty prior_readable_down_osds, clearing ub 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:01.652+0000 7fe2c425f700 1 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963942827s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:01.652+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963942827s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:01.652769+0000 0 0.000000 2019-12-07T19:34:01.652+0000 7fe2c6a64700 15 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] publish_stats_to_osd 23:34 2019-12-07T19:34:01.652+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963942827s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963942827s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 3(5) notify: (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973692310s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.967042856s) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.3(5) 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973692310s) 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963942827s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MQuery 1.fs1 from 3(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:14.607122+0000, must = 0 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5 unchanged 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 3 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 24 send_incremental_map 23 -> 24 to 0x558e818d7c00 v2:172.21.15.131:6823/37078 2019-12-07T19:34:01.652+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.fs0 1.fs0 (query:24 sent:24 1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963942827s) 1->0 ([18,23] all_participants=0(1),1(4),3(0),4(3),5(2),6(5) intervals=([18,23] acting 0(1),1(4),3(0),4(3),5(2),6(5)))) e24/24) v1 -- 0x558e81229c00 con 0x558e818d7000 2019-12-07T19:34:01.652+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(24..24 src has 1..24) v4 -- 0x558e83ff0000 con 0x558e818d7c00 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 24 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 23 -> 24 (shared) 2019-12-07T19:34:01.652+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.2s2 1.2s2 query(info 0'0 epoch_sent 24) e24/24) v1 -- 0x558e8402fb80 con 0x558e818d7c00 2019-12-07T19:34:01.652+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_query2(1.2s5 1.2s5 query(info 0'0 epoch_sent 24) e24/24) v1 -- 0x558e83b08000 con 0x558e818d7000 2019-12-07T19:34:01.652+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.fs0 1.fs0 (query:24 sent:24 1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) 1->0 ([18,23] all_participants=0(1),1(4),3(0),4(3),5(2),6(5) intervals=([18,23] acting 0(1),1(4),3(0),4(3),5(2),6(5)))) e24/24) v1 -- 0x558e8404a000 con 0x558e818d7000 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 24 send_incremental_map 23 -> 24 to 0x558e818fe400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:01.652+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(24..24 src has 1..24) v4 -- 0x558e83f2b200 con 0x558e818fe400 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e83c56600 seq 18 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 24 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 23 -> 24 (shared) 2019-12-07T19:34:01.652+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:01.652+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_query2(1.2s1 1.2s1 query(info 0'0 epoch_sent 24) e24/24) v1 -- 0x558e83b09680 con 0x558e818fe400 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 24 send_incremental_map 23 -> 24 to 0x558e818d6800 v2:172.21.15.109:6802/36936 2019-12-07T19:34:01.652+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(24..24 src has 1..24) v4 -- 0x558e83f2afc0 con 0x558e818d6800 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56600 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 24 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 23 -> 24 (shared) 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56600 onodes shared_blobs 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56600 allocated 0x[] released 0x[] 2019-12-07T19:34:01.652+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 24) e24/24) v1 -- 0x558e811dcc80 con 0x558e818d6800 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 prepare 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56600 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 io_done 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e81a0bb00 seq 20 2019-12-07T19:34:01.652+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.7s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.7s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558e8122e000 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0bb00 cost 5441 (1 ios * 4000 + 1441 bytes) 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0bb00 onodes shared_blobs 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 prepare 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0bb00 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 io_done 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 23 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.7s3 23 -> 24 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 24 queue_want_up_thru now 24 (was 18), currently 18 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 24 send_alive up_thru currently 18 want 24 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 24 send_alive want 24 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 23 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] handle_advance_map [1,2147483647,4,0,7,2]/[1,2147483647,4,0,7,2] -- 1/1 2019-12-07T19:34:01.652+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- osd_alive(want up_thru 24 have 24) v1 -- 0x558e819b1a40 con 0x558e80f39000 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] new interval newup [1,2147483647,4,0,7,2] newacting [1,2147483647,4,0,7,2] 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)> waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) pg 0x558e81075000 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.652+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:01.652997+0000 15 0.000288 2019-12-07T19:34:01.652+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:01.653006+0000 0 0.000000 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.5(4) 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 2019-12-07T19:34:01.652+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:01.653014+0000 0 0.000000 2019-12-07T19:34:01.652+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:20.148798+0000, must = 0 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=18 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5 unchanged 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=24 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 24 queue_want_up_thru want 24 <= queued 24, currently 18 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=24 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973540341s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=24 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)> waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973540341s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) pg 0x558e81075000 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=24 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] new interval newup [1,2147483647,4,0,7,2] newacting [1,2147483647,4,0,7,2] 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973540341s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=24 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,4,0,7,2]p1(0) r=3 lpr=24 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.4(1) 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973540341s) 2019-12-07T19:34:01.652+0000 7fe2c425f700 15 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:16.580480+0000, must = 0 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5 unchanged 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 4 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(18-23 up [1,6,4,0,7,2](1) acting [1,6,4,0,7,2](1)) up_thru 18 up_from 13 last_epoch_clean 19 interval(18-23 up [1,6,4,0,7,2](1) acting [1,6,4,0,7,2](1) maybe_went_rw) : primary up 13-18 includes interval 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 24 queue_want_up_thru want 24 <= queued 24, currently 18 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 1(2) notify: (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.970353093s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)> waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 1(2) notify: (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.970353093s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) pg 0x558e81075000 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] noting past ([18,23] all_participants=0(3),1(0),2(5),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4))) 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 1(2) notify: (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.970353093s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.1(2) 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.970353093s) 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:18.968661+0000, must = 0 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active pruub 36.073818636s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.073818636s (mnow 25.107824671s + 10.965993965s) 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5 unchanged 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active pruub 36.073818636s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active pruub 36.073818636s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 1 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Common peer features: 3f01cfb8ffadffff 2019-12-07T19:34:01.652+0000 7fe2c425f700 1 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'9 active pruub 36.073818636s@ mbc={} ps=[2~1]] start_peering_interval up [1,6,4,0,7,2] -> [1,2147483647,4,0,7,2], acting [1,6,4,0,7,2] -> [1,2147483647,4,0,7,2], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 3 -> 3, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Common acting features: 3f01cfb8ffadffff 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.073818636s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Common upacting features: 3f01cfb8ffadffff 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.073818636s@ mbc={} ps=[2~1]] release_backoffs [1:e0000000::::0,1:f0000000::::head) 2019-12-07T19:34:01.652+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] exit Started/Primary/Peering/GetInfo 2019-12-07T19:34:01.653175+0000 4 0.000804 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.073818636s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.652+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] enter Started/Primary/Peering/GetLog 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.073818636s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.073818636s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.0(0) 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.073818636s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.1(2) 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.970353093s) 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.073818636s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.3(5) 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973692310s) 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.073818636s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.4(1) 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.973540341s) 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2554 < min_flush_size 524288 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.073818636s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.5(4) 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.971221456s) 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.073818636s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.073818636s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3860398080 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown pruub 36.073818636s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] For position 0: selecting up[i]: 0(0) For position 1: selecting up[i]: 4(1) For position 2: selecting up[i]: 1(2) For position 3: failed to fill position 3 For position 4: selecting up[i]: 5(4) For position 5: selecting up[i]: 3(5) 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998678) bluestore 86 GiB free (0.959971), bluefs_ratio 0.0399508 2019-12-07T19:34:01.652+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2577 < min_flush_size 524288 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2577 < min_flush_size 524288 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x6bf15~a11 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x6bf15 0x6bf15~a11 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3bf15 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xf15 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x926 and padding block with 0x6da 2019-12-07T19:34:01.652+0000 7fe2ca26b700 10 osd.0 24 send_incremental_map 23 -> 24 to 0x558e818fe400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993965s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.652+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(24..24 src has 1..24) v4 -- 0x558e83fd0d80 con 0x558e818fe400 2019-12-07T19:34:01.652+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965834555s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:01.653296+0000 1 0.000279 2019-12-07T19:34:01.652+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965834555s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:01.652+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965834555s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:01.652+0000 7fe2c425f700 1 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965834555s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:01.652+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:24 sent:24 1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961940673s) 3->0 ([18,23] all_participants=0(3),1(4),2(2),4(0),5(5),6(1) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e24/24) v1 -- 0x558e83c48a00 con 0x558e818fe400 2019-12-07T19:34:01.652+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965834555s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:01.653324+0000 0 0.000000 2019-12-07T19:34:01.652+0000 7fe2c425f700 5 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965834555s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965834555s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.652+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:24 sent:24 1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) 3->0 ([18,23] all_participants=0(3),1(4),2(2),4(0),5(5),6(1) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e24/24) v1 -- 0x558e83ebd100 con 0x558e818fe400 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:01.652+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 99 ==== pg_query2(1.es4 1.es4 query(info 0'0 epoch_sent 24) e24/24) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8191bb80 con 0x558e818d7c00 2019-12-07T19:34:01.652+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:34:01.652+0000 7fe2e7d3c700 15 osd.0 24 enqueue_peering_evt 1.es4 epoch_sent: 24 epoch_requested: 24 MQuery 1.es4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.652+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e81a0b800 seq 24 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965834555s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MQuery 1.7s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.652+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.es4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:01.652+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:01.652+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:01.652+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:01.652+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:01.652+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:24 sent:24 1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965834555s) 3->0 ([18,23] all_participants=0(3),1(0),2(5),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e24/24) v1 -- 0x558e8404a700 con 0x558e818d7c00 2019-12-07T19:34:01.652+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:01.652+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b800 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:01.652+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b800 onodes shared_blobs 2019-12-07T19:34:01.652+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b800 allocated 0x[] released 0x[] 2019-12-07T19:34:01.652+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 prepare 2019-12-07T19:34:01.652+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:24 sent:24 1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) 3->0 ([18,23] all_participants=0(3),1(0),2(5),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e24/24) v1 -- 0x558e8404ae00 con 0x558e818d7c00 2019-12-07T19:34:01.652+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b800 2019-12-07T19:34:01.652+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 io_done 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e83c56900 seq 23 2019-12-07T19:34:01.652+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.652+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:01.652+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558e81928000 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56900 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:01.652+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56900 onodes shared_blobs 2019-12-07T19:34:01.652+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56900 allocated 0x[] released 0x[] 2019-12-07T19:34:01.652+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 prepare 2019-12-07T19:34:01.652+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.es4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56900 2019-12-07T19:34:01.652+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 io_done 2019-12-07T19:34:01.652+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558e8122e000 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558e81a9b000 2019-12-07T19:34:01.652+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 54 ==== pg_notify2(1.2s0 1.2s0 (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965899965s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e24/24) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818fe400 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.652+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:01.652+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:01.651+0000 7fe2c4a60700 10 osd.0 pg_epoch: 23 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.652+0000 7fe2c4a60700 20 osd.0:6.update_pg_epoch 1.es4 23 -> 24 2019-12-07T19:34:01.652+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 21 ==== pg_notify2(1.2s0 1.2s0 (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993709s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e24/24) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e8122b800 con 0x558e818d6800 2019-12-07T19:34:01.652+0000 7fe2e6d3a700 15 osd.0 24 enqueue_peering_evt 1.2s0 epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965899965s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.652+0000 7fe2c4a60700 10 osd.0 pg_epoch: 23 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] handle_advance_map [1,3,4,5,0,2147483647]/[1,3,4,5,0,2147483647] -- 1/1 2019-12-07T19:34:01.652+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965899965s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) 2019-12-07T19:34:01.652+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.652+0000 7fe2e753b700 15 osd.0 24 enqueue_peering_evt 1.2s0 epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993709s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.652+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993709s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) 2019-12-07T19:34:01.652+0000 7fe2c4a60700 20 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] new interval newup [1,3,4,5,0,2147483647] newacting [1,3,4,5,0,2147483647] 2019-12-07T19:34:01.652+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:01.652+0000 7fe2c4a60700 5 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:01.653545+0000 15 0.000169 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x6c926 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:01.652+0000 7fe2c4a60700 5 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:01.653555+0000 0 0.000000 2019-12-07T19:34:01.652+0000 7fe2c4a60700 5 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:01.653563+0000 0 0.000000 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000363257s (0.000000088s flush + 0.000363169s kv commit) 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_async_recovery_ec candidates by cost are: 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:01.652+0000 7fe2c4a60700 5 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_async_recovery_ec result want=[0,4,1,2147483647,5,3] async_recovery= 2019-12-07T19:34:01.652+0000 7fe2c4a60700 20 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=18 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:34:01.652+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=24 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2538 < min_flush_size 524288 2019-12-07T19:34:01.652+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=24 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] acting_recovery_backfill is 0(0),1(2),3(5),4(1),5(4) 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2561 < min_flush_size 524288 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2561 < min_flush_size 524288 2019-12-07T19:34:01.652+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=24 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting want=[0,4,1,2147483647,5,3] backfill_targets= async_recovery_targets= 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x6c926~a01 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.652+0000 7fe2c4a60700 20 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=24 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] new interval newup [1,3,4,5,0,2147483647] newacting [1,3,4,5,0,2147483647] 2019-12-07T19:34:01.652+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x6c926 0x6c926~a01 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: leaving GetLog 2019-12-07T19:34:01.652+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=24 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3c926 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x926 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:01.652+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x327 and padding block with 0xcd9 2019-12-07T19:34:01.652+0000 7fe2c4a60700 20 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,6]p1(0) r=4 lpr=24 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.652+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] exit Started/Primary/Peering/GetLog 2019-12-07T19:34:01.653614+0000 0 0.000000 2019-12-07T19:34:01.652+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] enter Started/Primary/Peering/GetMissing 2019-12-07T19:34:01.652+0000 7fe2c4a60700 15 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: osd.1(2) has no missing, identical log 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: osd.3(5) has no missing, identical log 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56600,0x558e81a0bb00> 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:01.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 kv_submitted 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56600 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: osd.4(1) has no missing, identical log 2019-12-07T19:34:01.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 finishing 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56600 onodes 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56600 done 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:01.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56600 [] 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: osd.5(4) has no missing, identical log 2019-12-07T19:34:01.652+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:01.652+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(18-23 up [1,3,4,5,0,6](1) acting [1,3,4,5,0,6](1)) up_thru 18 up_from 13 last_epoch_clean 19 interval(18-23 up [1,3,4,5,0,6](1) acting [1,3,4,5,0,6](1) maybe_went_rw) : primary up 13-18 includes interval 2019-12-07T19:34:01.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 kv_submitted 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0bb00 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: still need up_thru update before going active 2019-12-07T19:34:01.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 finishing 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0bb00 onodes 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0bb00 done 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:01.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0bb00 [] 2019-12-07T19:34:01.652+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:01.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:01.652+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] exit Started/Primary/Peering/GetMissing 2019-12-07T19:34:01.653663+0000 0 0.000000 2019-12-07T19:34:01.652+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] noting past ([18,23] all_participants=0(4),1(0),3(1),4(2),5(3),6(5) intervals=([18,23] acting 0(4),1(0),3(1),4(2),5(3),6(5))) 2019-12-07T19:34:01.652+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c6a64700 5 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] enter Started/Primary/Peering/WaitUpThru 2019-12-07T19:34:01.652+0000 7fe2c4a60700 20 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:01.652+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.as3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:01.652+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e81a0a600 seq 21 2019-12-07T19:34:01.652+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.as3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558e8193b000 2019-12-07T19:34:01.652+0000 7fe2c4a60700 20 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:01.652+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a600 cost 4032 (1 ios * 4000 + 32 bytes) 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a600 onodes shared_blobs 2019-12-07T19:34:01.652+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a600 allocated 0x[] released 0x[] 2019-12-07T19:34:01.652+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:01.653+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 prepare 2019-12-07T19:34:01.653+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a600 2019-12-07T19:34:01.653+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 io_done 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'4 active pruub 36.072387811s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.072387811s (mnow 25.108387523s + 10.964000288s) 2019-12-07T19:34:01.653+0000 7fe2c6a64700 10 osd.0 24 queue_want_up_thru want 24 <= queued 24, currently 18 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'4 active pruub 36.072387811s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:01.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'4 active pruub 36.072387811s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_advance_map: 24 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.as3 23 -> 24 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 23 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_advance_map [1,2147483647,2,0,5,4]/[1,2147483647,2,0,5,4] -- 1/1 2019-12-07T19:34:01.653+0000 7fe2c4a60700 1 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'4 active pruub 36.072387811s@ mbc={} ps=[2~1]] start_peering_interval up [1,3,4,5,0,6] -> [1,3,4,5,0,2147483647], acting [1,3,4,5,0,6] -> [1,3,4,5,0,2147483647], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown pruub 36.072387811s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] new interval newup [1,2147483647,2,0,5,4] newacting [1,2147483647,2,0,5,4] 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown pruub 36.072387811s@ mbc={} ps=[2~1]] release_backoffs [1:70000000::::0,1:80000000::::head) 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown pruub 36.072387811s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.653+0000 7fe2caa6c700 5 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:01.653748+0000 15 0.000155 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown pruub 36.072387811s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown pruub 36.072387811s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:01.653+0000 7fe2caa6c700 5 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:01.653756+0000 0 0.000000 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown pruub 36.072387811s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:01.653+0000 7fe2caa6c700 5 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:01.653763+0000 0 0.000000 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown pruub 36.072387811s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:01.653+0000 7fe2caa6c700 5 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown pruub 36.072387811s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown pruub 36.072387811s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=24 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown pruub 36.072387811s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=24 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown pruub 36.072387811s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=24 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown pruub 36.072387811s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=24 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] new interval newup [1,2147483647,2,0,5,4] newacting [1,2147483647,2,0,5,4] 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.653+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=24 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,6,2,0,5,4]p1(0) r=3 lpr=24 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] set_last_peering_reset 24 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.653+0000 7fe2caa6c700 15 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 osd.0 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964000288s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(18-23 up [1,6,2,0,5,4](1) acting [1,6,2,0,5,4](1)) up_thru 18 up_from 13 last_epoch_clean 19 interval(18-23 up [1,6,2,0,5,4](1) acting [1,6,2,0,5,4](1) maybe_went_rw) : primary up 13-18 includes interval 2019-12-07T19:34:01.653+0000 7fe2c4a60700 5 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963850027s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:01.653851+0000 1 0.000289 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] noting past ([18,23] all_participants=0(3),1(0),2(2),4(5),5(4),6(1) intervals=([18,23] acting 0(3),1(0),2(2),4(5),5(4),6(1))) 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:01.653+0000 7fe2c4a60700 5 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963850027s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:01.653+0000 7fe2c4a60700 5 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963850027s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:01.653+0000 7fe2c4a60700 1 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963850027s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:01.653+0000 7fe2c4a60700 5 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963850027s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:01.653886+0000 0 0.000000 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'2 active pruub 36.073166239s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.073166239s (mnow 25.108577474s + 10.964588765s) 2019-12-07T19:34:01.653+0000 7fe2c4a60700 5 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963850027s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'2 active pruub 36.073166239s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963850027s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'2 active pruub 36.073166239s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:01.653+0000 7fe2caa6c700 1 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'2 active pruub 36.073166239s@ mbc={} ps=[2~1]] start_peering_interval up [1,6,2,0,5,4] -> [1,2147483647,2,0,5,4], acting [1,6,2,0,5,4] -> [1,2147483647,2,0,5,4], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 3 -> 3, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown pruub 36.073166239s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963850027s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown pruub 36.073166239s@ mbc={} ps=[2~1]] release_backoffs [1:50000000::::0,1:60000000::::head) 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown pruub 36.073166239s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown pruub 36.073166239s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.653+0000 7fe2c4a60700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.es0 1.es0 (query:24 sent:24 1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963850027s) 4->0 ([18,23] all_participants=0(4),1(0),3(1),4(2),5(3),6(5) intervals=([18,23] acting 0(4),1(0),3(1),4(2),5(3),6(5)))) e24/24) v1 -- 0x558e8404b500 con 0x558e818d7c00 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown pruub 36.073166239s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown pruub 36.073166239s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e83c57500 seq 14 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown pruub 36.073166239s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:01.653+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown pruub 36.073166239s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown pruub 36.073166239s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c57500 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c57500 onodes shared_blobs 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c57500 allocated 0x[] released 0x[] 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown pruub 36.073166239s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 prepare 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c57500 2019-12-07T19:34:01.653+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 io_done 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown pruub 36.073166239s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:01.653+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown pruub 36.073166239s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:01.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(18-23 up [1,4,5,0,7,6](1) acting [1,4,5,0,7,6](1)) up_thru 18 up_from 13 last_epoch_clean 19 interval(18-23 up [1,4,5,0,7,6](1) acting [1,4,5,0,7,6](1) maybe_went_rw) : primary up 13-18 includes interval 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.653+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] noting past ([18,23] all_participants=0(3),1(0),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4))) 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active pruub 36.072371497s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.072371497s (mnow 25.108733672s + 10.963637825s) 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964588765s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active pruub 36.072371497s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active pruub 36.072371497s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:01.653+0000 7fe2caa6c700 5 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964424715s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:01.654057+0000 1 0.000281 2019-12-07T19:34:01.653+0000 7fe2cb26d700 1 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 luod=0'0 crt=21'6 active pruub 36.072371497s@ mbc={} ps=[2~1]] start_peering_interval up [1,4,5,0,7,6] -> [1,4,5,0,7,2147483647], acting [1,4,5,0,7,6] -> [1,4,5,0,7,2147483647], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 3 -> 3, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:01.653+0000 7fe2caa6c700 5 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964424715s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.072371497s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:01.653+0000 7fe2caa6c700 5 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964424715s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:01.653+0000 7fe2caa6c700 1 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964424715s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.072371497s@ mbc={} ps=[2~1]] release_backoffs [1:90000000::::0,1:a0000000::::head) 2019-12-07T19:34:01.653+0000 7fe2caa6c700 5 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964424715s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:01.654085+0000 0 0.000000 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.072371497s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.072371497s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.653+0000 7fe2caa6c700 5 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964424715s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.072371497s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964424715s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.072371497s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.072371497s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.072371497s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.072371497s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964424715s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MQuery 1.as3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.072371497s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.072371497s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown pruub 36.072371497s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:01.653+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:34:01.653+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.as0 1.as0 (query:24 sent:24 1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964424715s) 3->0 ([18,23] all_participants=0(3),1(0),2(2),4(5),5(4),6(1) intervals=([18,23] acting 0(3),1(0),2(2),4(5),5(4),6(1)))) e24/24) v1 -- 0x558e8404bc00 con 0x558e818d7c00 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 osd.0 24 advance_pg new pool opts old pool opts 2019-12-07T19:34:01.653+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.as0 1.as0 (query:24 sent:24 1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) 3->0 ([18,23] all_participants=0(3),1(0),2(2),4(5),5(4),6(1) intervals=([18,23] acting 0(3),1(0),2(2),4(5),5(4),6(1)))) e24/24) v1 -- 0x558e8404c300 con 0x558e818d7c00 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] handle_activate_map: 24 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e8106c900 seq 10 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963637825s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:01.653+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:01.653+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963490206s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:01.654193+0000 1 0.001975 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c900 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c900 onodes shared_blobs 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c900 allocated 0x[] released 0x[] 2019-12-07T19:34:01.653+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963490206s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 prepare 2019-12-07T19:34:01.653+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963490206s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c900 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 io_done 2019-12-07T19:34:01.653+0000 7fe2cb26d700 1 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963490206s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.653+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963490206s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:01.654221+0000 0 0.000000 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558e81075000 2019-12-07T19:34:01.653+0000 7fe2cb26d700 5 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963490206s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963490206s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 24 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 24 queue_want_up_thru want 24 <= queued 24, currently 18 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965899965s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)> waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965899965s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) pg 0x558e81075000 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963490206s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MQuery 1.9s3 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 4(1) notify: (query:24 sent:24 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965899965s) 1->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:01.653+0000 7fe2caa6c700 7 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: handle_pg_notify from osd.4(1) 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got dup osd.4(1) info 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965899965s), identical to ours 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 24 queue_want_up_thru want 24 <= queued 24, currently 18 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993709s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.653+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:24 sent:24 1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963490206s) 3->0 ([18,23] all_participants=0(3),1(0),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4)))) e24/24) v1 -- 0x558e83ebca00 con 0x558e818d7c00 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)> waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993709s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) pg 0x558e81075000 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 5(4) notify: (query:24 sent:24 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993709s) 4->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.653+0000 7fe2caa6c700 7 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: handle_pg_notify from osd.5(4) 2019-12-07T19:34:01.653+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:24 sent:24 1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) 3->0 ([18,23] all_participants=0(3),1(0),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4)))) e24/24) v1 -- 0x558e83c9c000 con 0x558e818d7c00 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got dup osd.5(4) info 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965993709s), identical to ours 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 24 queue_want_up_thru want 24 <= queued 24, currently 18 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83d4c900 seq 14 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:01.653+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:01.653+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 15 ==== osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 ==== 302+0+4 (crc 0 0 0) 0x558e83aae580 con 0x558e83d68000 2019-12-07T19:34:01.653+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:01.653+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MQuery 1.es4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24)) prio 255 cost 10 e24) pg 0x558e81a98000 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:01.653+0000 7fe2e6d3a700 15 osd.0 24 enqueue_op 0x558e819431e0 prio 63 cost 4 latency 0.000016 epoch 24 osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:01.653+0000 7fe2c8a68700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963850027s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MQuery 1.es4 from 1(0) query_epoch 24 query: query(info 0'0 epoch_sent 24) 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c900 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c900 onodes shared_blobs 2019-12-07T19:34:01.653+0000 7fe2c8a68700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c900 allocated 0x[] released 0x[] 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 prepare 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c900 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 io_done 2019-12-07T19:34:01.653+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558e819e4000 2019-12-07T19:34:01.653+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.es0 1.es0 (query:24 sent:24 1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) 4->0 ([18,23] all_participants=0(4),1(0),3(1),4(2),5(3),6(5) intervals=([18,23] acting 0(4),1(0),3(1),4(2),5(3),6(5)))) e24/24) v1 -- 0x558e83c9dc00 con 0x558e818d7c00 2019-12-07T19:34:01.653+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e24) 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.653+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:01.653+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:01.653+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:01.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558e8192b000 2019-12-07T19:34:01.653+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 16 ==== osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 ==== 264+0+4 (crc 0 0 0) 0x558e83ca6000 con 0x558e83d68000 2019-12-07T19:34:01.653+0000 7fe2c7265700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.653+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:01.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:01.653+0000 7fe2e6d3a700 15 osd.0 24 enqueue_op 0x558e81943080 prio 63 cost 4 latency 0.000016 epoch 24 osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 2019-12-07T19:34:01.653+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e24) 2019-12-07T19:34:01.653+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 17 ==== osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 ==== 264+0+4 (crc 0 0 0) 0x558e83ca62c0 con 0x558e83d68000 2019-12-07T19:34:01.653+0000 7fe2e6d3a700 15 osd.0 24 enqueue_op 0x558e819438c0 prio 63 cost 4 latency 0.000007 epoch 24 osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 2019-12-07T19:34:01.653+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e24) 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e24) queued 2019-12-07T19:34:01.653+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x6d327 2019-12-07T19:34:01.653+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:01.653+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:01.653+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e24) pg 0x558e81075000 2019-12-07T19:34:01.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000873129s (0.000000061s flush + 0.000873068s kv commit) 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 24 dequeue_op 0x558e819431e0 prio 63 cost 4 latency 0.000132 osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] 2019-12-07T19:34:01.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 24 maybe_share_map con 0x558e83d68000 172.21.15.131:0/224485236 map epoch 23 -> 24 (as per caller) 2019-12-07T19:34:01.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 24 dequeue_op 0x558e819431e0 finish 2019-12-07T19:34:01.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e24) queued 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e24) pg 0x558e81075000 2019-12-07T19:34:01.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:01.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2561 < min_flush_size 524288 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 24 dequeue_op 0x558e81943080 prio 63 cost 4 latency 0.000088 osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] 2019-12-07T19:34:01.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 24 dequeue_op 0x558e81943080 finish 2019-12-07T19:34:01.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3830 < min_flush_size 524288 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e24) queued 2019-12-07T19:34:01.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3853 < min_flush_size 524288 2019-12-07T19:34:01.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3853 < min_flush_size 524288 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e24) pg 0x558e81075000 2019-12-07T19:34:01.653+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x6d327~f0d to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 24 dequeue_op 0x558e819438c0 prio 63 cost 4 latency 0.000084 osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] 2019-12-07T19:34:01.653+0000 7fe2caa6c700 10 osd.0 24 dequeue_op 0x558e819438c0 finish 2019-12-07T19:34:01.653+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x6d327 0x6d327~f0d to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:01.653+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:01.653+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~40000 x_off 0x3d327 2019-12-07T19:34:01.653+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x327 2019-12-07T19:34:01.653+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:01.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:01.653+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x234 and padding block with 0xdcc 2019-12-07T19:34:01.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 NullEvt) prio 255 cost 10 e24) pg 0x558e8193b000 2019-12-07T19:34:01.653+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 NullEvt 2019-12-07T19:34:01.653+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:01.653+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:01.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:01.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0b800,0x558e83c56900> 2019-12-07T19:34:01.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:01.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 kv_submitted 2019-12-07T19:34:01.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b800 2019-12-07T19:34:01.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 finishing 2019-12-07T19:34:01.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b800 onodes 2019-12-07T19:34:01.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 done 2019-12-07T19:34:01.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:34:01.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b800 [] 2019-12-07T19:34:01.653+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:01.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 kv_submitted 2019-12-07T19:34:01.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56900 2019-12-07T19:34:01.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 finishing 2019-12-07T19:34:01.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56900 onodes 2019-12-07T19:34:01.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56900 done 2019-12-07T19:34:01.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:01.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56900 [] 2019-12-07T19:34:01.653+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:01.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:01.653+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 55 ==== pg_notify2(1.2s0 1.2s0 (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965399477s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e24/24) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e8404a000 con 0x558e818d7000 2019-12-07T19:34:01.653+0000 7fe2e7d3c700 15 osd.0 24 enqueue_peering_evt 1.2s0 epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 3(5) notify: (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965399477s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.653+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 3(5) notify: (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965399477s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) 2019-12-07T19:34:01.653+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 100 ==== pg_notify2(1.2s0 1.2s0 (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965139759s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e24/24) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e8404ae00 con 0x558e818d7c00 2019-12-07T19:34:01.654+0000 7fe2e7d3c700 15 osd.0 24 enqueue_peering_evt 1.2s0 epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 1(2) notify: (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965139759s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.654+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 1(2) notify: (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965139759s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) 2019-12-07T19:34:01.654+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x6e234 2019-12-07T19:34:01.654+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:01.654+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:01.654+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:01.654+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:01.654+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 4 cleaned 0 in 0.000329543s (0.000000062s flush + 0.000329481s kv commit) 2019-12-07T19:34:01.654+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0a600,0x558e83c57500,0x558e8106c900,0x558e83d4c900> 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 kv_submitted 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a600 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 finishing 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a600 onodes 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a600 done 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a600 [] 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 kv_submitted 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c57500 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 finishing 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c57500 onodes 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57500 done 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c57500 [] 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 kv_submitted 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c900 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c900 finishing 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c900 onodes 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c900 done 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c900 [] 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 kv_submitted 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c900 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 finishing 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c900 onodes 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c900 done 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c900 [] 2019-12-07T19:34:01.654+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:01.654+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:01.654+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:01.654+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:01.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 3(5) notify: (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965399477s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)> waiting <> waiting_peering {} 2019-12-07T19:34:01.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 3(5) notify: (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965399477s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) pg 0x558e81075000 2019-12-07T19:34:01.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 3(5) notify: (query:24 sent:24 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965399477s) 5->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.654+0000 7fe2c6a64700 7 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: handle_pg_notify from osd.3(5) 2019-12-07T19:34:01.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got dup osd.3(5) info 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965399477s), identical to ours 2019-12-07T19:34:01.654+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:01.654+0000 7fe2c6a64700 10 osd.0 24 queue_want_up_thru want 24 <= queued 24, currently 18 2019-12-07T19:34:01.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:01.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 1(2) notify: (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965139759s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) queued 2019-12-07T19:34:01.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24)> waiting <> waiting_peering {} 2019-12-07T19:34:01.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 1(2) notify: (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965139759s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e24) pg 0x558e81075000 2019-12-07T19:34:01.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 24 epoch_requested: 24 MNotifyRec 1.2s0 from 1(2) notify: (query:24 sent:24 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965139759s) 2->0 ([18,23] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:01.654+0000 7fe2c6a64700 7 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: handle_pg_notify from osd.1(2) 2019-12-07T19:34:01.654+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got dup osd.1(2) info 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965139759s), identical to ours 2019-12-07T19:34:01.654+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:01.654+0000 7fe2c6a64700 10 osd.0 24 queue_want_up_thru want 24 <= queued 24, currently 18 2019-12-07T19:34:01.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:01.654+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:01.654+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:01.654+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:01.654+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:01.654+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:01.654+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:01.698+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.701+0000 7fe2e2ca2700 10 osd.0 24 tick_without_osd_lock 2019-12-07T19:34:01.701+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fd5d000/0x40000000/0x1680000000, data 0x27926e/0x2a1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:01.701+0000 7fe2e2ca2700 25 osd.0 24 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453695+0000 last_rx_front 2019-12-07T19:33:56.453704+0000 2019-12-07T19:34:01.701+0000 7fe2e2ca2700 25 osd.0 24 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453559+0000 last_rx_front 2019-12-07T19:33:56.453627+0000 2019-12-07T19:34:01.701+0000 7fe2e2ca2700 25 osd.0 24 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453575+0000 last_rx_front 2019-12-07T19:33:56.453596+0000 2019-12-07T19:34:01.701+0000 7fe2e2ca2700 25 osd.0 24 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453839+0000 last_rx_front 2019-12-07T19:33:56.453807+0000 2019-12-07T19:34:01.701+0000 7fe2e2ca2700 25 osd.0 24 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453763+0000 last_rx_front 2019-12-07T19:33:56.453773+0000 2019-12-07T19:34:01.701+0000 7fe2e2ca2700 25 osd.0 24 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:33:56.453331+0000 last_rx_back 2019-12-07T19:33:56.453789+0000 last_rx_front 2019-12-07T19:33:56.453733+0000 2019-12-07T19:34:01.701+0000 7fe2e2ca2700 20 osd.0 24 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:01.701+0000 7fe2e2ca2700 10 osd.0 24 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:01.701+0000 7fe2e2ca2700 20 osd.0 24 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:01.701+0000 7fe2e2ca2700 10 osd.0 24 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:01.705+0000 7fe2e4535700 10 osd.0 24 tick 2019-12-07T19:34:01.705+0000 7fe2e4535700 10 osd.0 24 maybe_update_heartbeat_peers updating 2019-12-07T19:34:01.705+0000 7fe2e4535700 10 osd.0 24 adding neighbor peer osd.1 2019-12-07T19:34:01.705+0000 7fe2e4535700 10 osd.0 24 adding neighbor peer osd.5 2019-12-07T19:34:01.705+0000 7fe2e4535700 10 osd.0 24 adding neighbor peer osd.7 2019-12-07T19:34:01.705+0000 7fe2e4535700 10 osd.0 24 adding random peer osd.3 2019-12-07T19:34:01.705+0000 7fe2e4535700 10 osd.0 24 adding random peer osd.4 2019-12-07T19:34:01.705+0000 7fe2e4535700 10 osd.0 24 maybe_update_heartbeat_peers 6 peers, extras 1,2,3,4,5,7 2019-12-07T19:34:01.705+0000 7fe2e4535700 10 osd.0 24 do_waiters -- start 2019-12-07T19:34:01.705+0000 7fe2e4535700 10 osd.0 24 do_waiters -- finish 2019-12-07T19:34:01.705+0000 7fe2e4535700 20 osd.0 24 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:01.748+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.752+0000 7fe2c3a5e700 5 osd.0 24 heartbeat osd_stat(store_statfs(0x163fd5d000/0x40000000/0x1680000000, data 0x27926e/0x2a1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5,7] op hist [0,0,0,0,0,0,0,3]) 2019-12-07T19:34:01.752+0000 7fe2c3a5e700 20 osd.0 24 check_full_status cur ratio 2.86102e-05, physical ratio 2.86102e-05, new state none 2019-12-07T19:34:01.752+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.208340107s delta_ub 18446744073.707461837s) v5 -- 0x558e818ef200 con 0x558e818d7800 2019-12-07T19:34:01.752+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.208340107s delta_ub 18446744073.707461837s) v5 -- 0x558e8118ee00 con 0x558e818d7400 2019-12-07T19:34:01.752+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.208340107s delta_ub 18446744073.693426540s) v5 -- 0x558e8118fa00 con 0x558e817ff000 2019-12-07T19:34:01.753+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.208340107s delta_ub 18446744073.693426540s) v5 -- 0x558e8118e800 con 0x558e817ff400 2019-12-07T19:34:01.753+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.208340107s delta_ub 18446744073.706894104s) v5 -- 0x558e8118ea00 con 0x558e817ff800 2019-12-07T19:34:01.753+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.208340107s delta_ub 18446744073.706894104s) v5 -- 0x558e8118fc00 con 0x558e817ffc00 2019-12-07T19:34:01.753+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.208340107s delta_ub 0.005976560s) v5 -- 0x558e84052a00 con 0x558e8181a000 2019-12-07T19:34:01.753+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.208340107s delta_ub 0.005976560s) v5 -- 0x558e84052c00 con 0x558e8181a400 2019-12-07T19:34:01.753+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.208340107s delta_ub 18446744073.698198034s) v5 -- 0x558e84052e00 con 0x558e817fe000 2019-12-07T19:34:01.753+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.208340107s delta_ub 18446744073.698198034s) v5 -- 0x558e84053000 con 0x558e817fe400 2019-12-07T19:34:01.753+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.208340107s delta_ub 0.019588628s) v5 -- 0x558e84053200 con 0x558e817fe800 2019-12-07T19:34:01.753+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.208340107s delta_ub 0.019588628s) v5 -- 0x558e84053400 con 0x558e817fec00 2019-12-07T19:34:01.753+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6827/37078 2 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.210773192s delta_ub 0.002433085s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84028200 con 0x558e818d7400 2019-12-07T19:34:01.753+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6816/37076 6 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.211356677s delta_ub 0.003016570s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81955c00 con 0x558e817ff800 2019-12-07T19:34:01.753+0000 7fe2e6d3a700 25 osd.0 24 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:01.753648+0000 last_rx_back 2019-12-07T19:33:56.453695+0000 last_rx_front 2019-12-07T19:33:56.453704+0000 -> 2019-12-07T19:34:01.753931+0000 2019-12-07T19:34:01.753+0000 7fe2e6d3a700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002150083s,0.002433085s]) 2019-12-07T19:34:01.753+0000 7fe2e7d3c700 25 osd.0 24 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:01.753648+0000 last_rx_back 2019-12-07T19:33:56.453575+0000 -> 2019-12-07T19:34:01.753956+0000 last_rx_front 2019-12-07T19:33:56.453596+0000 2019-12-07T19:34:01.753+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6831/37078 2 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.210725483s delta_ub 0.002385376s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118f400 con 0x558e818d7800 2019-12-07T19:34:01.753+0000 7fe2e7d3c700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002708450s,0.003016570s]) 2019-12-07T19:34:01.753+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6806/37077 6 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.225567205s delta_ub 0.017227098s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84028200 con 0x558e817ff400 2019-12-07T19:34:01.753+0000 7fe2e6d3a700 25 osd.0 24 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:01.753648+0000 last_rx_back 2019-12-07T19:33:56.453559+0000 last_rx_front 2019-12-07T19:33:56.453627+0000 -> 2019-12-07T19:34:01.753971+0000 2019-12-07T19:34:01.753+0000 7fe2e6d3a700 10 osd.0 24 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 21 -> 24 (as per caller) 2019-12-07T19:34:01.753+0000 7fe2e6d3a700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016904045s,0.017227098s]) 2019-12-07T19:34:01.753+0000 7fe2e753b700 25 osd.0 24 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:01.753648+0000 last_rx_back 2019-12-07T19:33:56.453695+0000 -> 2019-12-07T19:34:01.753986+0000 last_rx_front 2019-12-07T19:34:01.753931+0000 2019-12-07T19:34:01.753+0000 7fe2e753b700 25 osd.0 24 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:01.753648+0000) and older pending ping(s) 2019-12-07T19:34:01.753+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6812/36938 6 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.203027857s delta_ub 18446744073.704239366s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81955c00 con 0x558e8181a400 2019-12-07T19:34:01.753+0000 7fe2e753b700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002047555s,0.002385376s]) 2019-12-07T19:34:01.753+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6814/36938 6 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.203075394s delta_ub 18446744073.704286903s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84028200 con 0x558e8181a000 2019-12-07T19:34:01.753+0000 7fe2e6d3a700 25 osd.0 24 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:01.753648+0000 last_rx_back 2019-12-07T19:33:56.453839+0000 -> 2019-12-07T19:34:01.754004+0000 last_rx_front 2019-12-07T19:33:56.453807+0000 2019-12-07T19:34:01.753+0000 7fe2e6d3a700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703931041s,18446744073.704286903s]) 2019-12-07T19:34:01.753+0000 7fe2e7d3c700 25 osd.0 24 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:01.753648+0000 last_rx_back 2019-12-07T19:34:01.754004+0000 last_rx_front 2019-12-07T19:33:56.453807+0000 -> 2019-12-07T19:34:01.754016+0000 2019-12-07T19:34:01.753+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6814/37076 6 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.211378204s delta_ub 0.003038097s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118f400 con 0x558e817ffc00 2019-12-07T19:34:01.753+0000 7fe2e7d3c700 25 osd.0 24 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:01.753648+0000) and older pending ping(s) 2019-12-07T19:34:01.753+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6830/36939 6 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.189323543s delta_ub 18446744073.690535052s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84028200 con 0x558e817fe800 2019-12-07T19:34:01.753+0000 7fe2e7d3c700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703871562s,18446744073.704239366s]) 2019-12-07T19:34:01.753+0000 7fe2e753b700 25 osd.0 24 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:01.753648+0000 last_rx_back 2019-12-07T19:34:01.753956+0000 last_rx_front 2019-12-07T19:33:56.453596+0000 -> 2019-12-07T19:34:01.754033+0000 2019-12-07T19:34:01.753+0000 7fe2e753b700 25 osd.0 24 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:01.753648+0000) and older pending ping(s) 2019-12-07T19:34:01.753+0000 7fe2e753b700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002652905s,0.003038097s]) 2019-12-07T19:34:01.753+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6828/36939 6 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.189269897s delta_ub 18446744073.690481406s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81955c00 con 0x558e817fec00 2019-12-07T19:34:01.753+0000 7fe2e7d3c700 25 osd.0 24 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:01.753648+0000 last_rx_back 2019-12-07T19:33:56.453789+0000 last_rx_front 2019-12-07T19:33:56.453733+0000 -> 2019-12-07T19:34:01.754044+0000 2019-12-07T19:34:01.753+0000 7fe2e7d3c700 10 osd.0 24 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 23 -> 24 (as per caller) 2019-12-07T19:34:01.753+0000 7fe2e7d3c700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690085385s,18446744073.690481406s]) 2019-12-07T19:34:01.753+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6808/37077 6 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.225613419s delta_ub 0.017273312s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118f400 con 0x558e817ff000 2019-12-07T19:34:01.753+0000 7fe2e6d3a700 25 osd.0 24 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:01.753648+0000 last_rx_back 2019-12-07T19:33:56.453789+0000 -> 2019-12-07T19:34:01.754054+0000 last_rx_front 2019-12-07T19:34:01.754044+0000 2019-12-07T19:34:01.753+0000 7fe2e6d3a700 25 osd.0 24 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:34:01.753648+0000) and older pending ping(s) 2019-12-07T19:34:01.753+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6806/36936 6 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.220165248s delta_ub 0.011825141s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81955c00 con 0x558e817fe000 2019-12-07T19:34:01.753+0000 7fe2e6d3a700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690128762s,18446744073.690535052s]) 2019-12-07T19:34:01.753+0000 7fe2e753b700 25 osd.0 24 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:01.753648+0000 last_rx_back 2019-12-07T19:33:56.453559+0000 -> 2019-12-07T19:34:01.754069+0000 last_rx_front 2019-12-07T19:34:01.753971+0000 2019-12-07T19:34:01.753+0000 7fe2e753b700 25 osd.0 24 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:01.753648+0000) and older pending ping(s) 2019-12-07T19:34:01.753+0000 7fe2e753b700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016852423s,0.017273312s]) 2019-12-07T19:34:01.753+0000 7fe2e7d3c700 25 osd.0 24 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:01.753648+0000 last_rx_back 2019-12-07T19:33:56.453763+0000 -> 2019-12-07T19:34:01.754082+0000 last_rx_front 2019-12-07T19:33:56.453773+0000 2019-12-07T19:34:01.753+0000 7fe2e7d3c700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011390756s,0.011825141s]) 2019-12-07T19:34:01.753+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6804/36936 6 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.753648+0000/25.208340107s send_stamp 25.220149261s delta_ub 0.011809154s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118f400 con 0x558e817fe400 2019-12-07T19:34:01.753+0000 7fe2e753b700 25 osd.0 24 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:01.753648+0000 last_rx_back 2019-12-07T19:34:01.754082+0000 last_rx_front 2019-12-07T19:33:56.453773+0000 -> 2019-12-07T19:34:01.754093+0000 2019-12-07T19:34:01.753+0000 7fe2e753b700 25 osd.0 24 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:01.753648+0000) and older pending ping(s) 2019-12-07T19:34:01.753+0000 7fe2e753b700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011364579s,0.011809154s]) 2019-12-07T19:34:01.798+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.848+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.898+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.948+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:01.952+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.3 172.21.15.131:0/37076 4 ==== osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.953087+0000/25.410599692s send_stamp 25.410599692s delta_ub 0.003038097s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e8118f400 con 0x558e818ff800 2019-12-07T19:34:01.952+0000 7fe2e753b700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002678513s,0.003038097s]) 2019-12-07T19:34:01.952+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.953087+0000/25.410599692s send_stamp 25.407921179s delta_ub 18446744073.706873103s) v5 -- 0x558e84053000 con 0x558e818ff800 2019-12-07T19:34:01.952+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.3 172.21.15.131:0/37076 4 ==== osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:01.953087+0000/25.410599692s send_stamp 25.410599692s delta_ub 0.003038097s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84028200 con 0x558e81984000 2019-12-07T19:34:01.952+0000 7fe2e6d3a700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002650828s,0.003038097s]) 2019-12-07T19:34:01.952+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:01.953087+0000/25.410599692s send_stamp 25.407948864s delta_ub 18446744073.706900788s) v5 -- 0x558e84029e00 con 0x558e81984000 2019-12-07T19:34:01.998+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.049+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.099+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.149+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.160+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.4 172.21.15.109:0/36938 2 ==== osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.160132+0000/25.609358819s send_stamp 25.609358819s delta_ub 18446744073.704286903s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83fef800 con 0x558e81985400 2019-12-07T19:34:02.160+0000 7fe2e6d3a700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703628582s,18446744073.704286903s]) 2019-12-07T19:34:02.160+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.160132+0000/25.609358819s send_stamp 25.615281853s delta_ub 0.005923034s) v5 -- 0x558e84029e00 con 0x558e81985400 2019-12-07T19:34:02.160+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.4 172.21.15.109:0/36938 2 ==== osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.160132+0000/25.609358819s send_stamp 25.609358819s delta_ub 18446744073.704286903s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81955c00 con 0x558e81985800 2019-12-07T19:34:02.160+0000 7fe2e7d3c700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703556832s,18446744073.704286903s]) 2019-12-07T19:34:02.160+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.160132+0000/25.609358819s send_stamp 25.615353603s delta_ub 0.005994784s) v5 -- 0x558e84053400 con 0x558e81985800 2019-12-07T19:34:02.199+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.231+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:02.249+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.253+0000 7fe2c3a5e700 5 osd.0 24 heartbeat osd_stat(store_statfs(0x163fd5d000/0x40000000/0x1680000000, data 0x27926e/0x2a1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5,7] op hist [0,0,0,0,0,0,0,0,0,0,3]) 2019-12-07T19:34:02.253+0000 7fe2c3a5e700 20 osd.0 24 check_full_status cur ratio 2.86102e-05, physical ratio 2.86102e-05, new state none 2019-12-07T19:34:02.253+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.708637204s delta_ub 18446744073.707504061s) v5 -- 0x558e84053600 con 0x558e818d7800 2019-12-07T19:34:02.253+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.708637204s delta_ub 18446744073.707504061s) v5 -- 0x558e84053800 con 0x558e818d7400 2019-12-07T19:34:02.253+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.708637204s delta_ub 18446744073.692699193s) v5 -- 0x558e84053a00 con 0x558e817ff000 2019-12-07T19:34:02.253+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.708637204s delta_ub 18446744073.692699193s) v5 -- 0x558e84053c00 con 0x558e817ff400 2019-12-07T19:34:02.253+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.708637204s delta_ub 18446744073.706900788s) v5 -- 0x558e84053e00 con 0x558e817ff800 2019-12-07T19:34:02.253+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.708637204s delta_ub 18446744073.706900788s) v5 -- 0x558e83ca8000 con 0x558e817ffc00 2019-12-07T19:34:02.253+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.708637204s delta_ub 0.005994784s) v5 -- 0x558e83ca8200 con 0x558e8181a000 2019-12-07T19:34:02.253+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.708637204s delta_ub 0.005994784s) v5 -- 0x558e83ca8400 con 0x558e8181a400 2019-12-07T19:34:02.253+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.708637204s delta_ub 18446744073.698187037s) v5 -- 0x558e83ca8600 con 0x558e817fe000 2019-12-07T19:34:02.253+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.708637204s delta_ub 18446744073.698187037s) v5 -- 0x558e83ca8800 con 0x558e817fe400 2019-12-07T19:34:02.253+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.708637204s delta_ub 0.019422854s) v5 -- 0x558e83ca8a00 con 0x558e817fe800 2019-12-07T19:34:02.253+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.708637204s delta_ub 0.019422854s) v5 -- 0x558e83ca8c00 con 0x558e817fec00 2019-12-07T19:34:02.253+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6814/37076 7 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.711624452s delta_ub 0.002987248s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83ca8000 con 0x558e817ffc00 2019-12-07T19:34:02.253+0000 7fe2e753b700 25 osd.0 24 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:01.753956+0000 last_rx_front 2019-12-07T19:34:01.754033+0000 -> 2019-12-07T19:34:02.254194+0000 2019-12-07T19:34:02.253+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6816/37076 7 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.711652312s delta_ub 0.003015108s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81955c00 con 0x558e817ff800 2019-12-07T19:34:02.253+0000 7fe2e753b700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002738135s,0.002987248s]) 2019-12-07T19:34:02.253+0000 7fe2e7d3c700 25 osd.0 24 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:01.753956+0000 -> 2019-12-07T19:34:02.254209+0000 last_rx_front 2019-12-07T19:34:02.254194+0000 2019-12-07T19:34:02.253+0000 7fe2e7d3c700 25 osd.0 24 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:02.253945+0000) and older pending ping(s) 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6806/37077 7 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.725860315s delta_ub 0.017223111s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81239c00 con 0x558e817ff400 2019-12-07T19:34:02.253+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6831/37078 3 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.711018622s delta_ub 0.002381418s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83ca8000 con 0x558e818d7800 2019-12-07T19:34:02.253+0000 7fe2e7d3c700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002751230s,0.003015108s]) 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 25 osd.0 24 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:01.754069+0000 last_rx_front 2019-12-07T19:34:01.753971+0000 -> 2019-12-07T19:34:02.254232+0000 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016936333s,0.017223111s]) 2019-12-07T19:34:02.253+0000 7fe2e753b700 25 osd.0 24 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:01.753986+0000 -> 2019-12-07T19:34:02.254252+0000 last_rx_front 2019-12-07T19:34:01.753931+0000 2019-12-07T19:34:02.253+0000 7fe2e753b700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002074832s,0.002381418s]) 2019-12-07T19:34:02.253+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6808/37077 7 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.725821669s delta_ub 0.017184465s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83ca8000 con 0x558e817ff000 2019-12-07T19:34:02.253+0000 7fe2e753b700 25 osd.0 24 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:01.754069+0000 -> 2019-12-07T19:34:02.254273+0000 last_rx_front 2019-12-07T19:34:02.254232+0000 2019-12-07T19:34:02.253+0000 7fe2e753b700 25 osd.0 24 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:02.253945+0000) and older pending ping(s) 2019-12-07T19:34:02.253+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6812/36938 7 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.703359780s delta_ub 18446744073.704274192s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81955c00 con 0x558e8181a400 2019-12-07T19:34:02.253+0000 7fe2e753b700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016856281s,0.017184465s]) 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6827/37078 3 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.711063790s delta_ub 0.002426586s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81239c00 con 0x558e818d7400 2019-12-07T19:34:02.253+0000 7fe2e7d3c700 25 osd.0 24 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:01.754004+0000 last_rx_front 2019-12-07T19:34:01.754016+0000 -> 2019-12-07T19:34:02.254290+0000 2019-12-07T19:34:02.253+0000 7fe2e7d3c700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703929506s,18446744073.704274192s]) 2019-12-07T19:34:02.253+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6804/36936 7 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.720476245s delta_ub 0.011839041s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83ca8000 con 0x558e817fe400 2019-12-07T19:34:02.253+0000 7fe2e753b700 25 osd.0 24 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:01.754082+0000 last_rx_front 2019-12-07T19:34:01.754093+0000 -> 2019-12-07T19:34:02.254301+0000 2019-12-07T19:34:02.253+0000 7fe2e753b700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011482947s,0.011839041s]) 2019-12-07T19:34:02.253+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6828/36939 7 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.689586599s delta_ub 18446744073.690501011s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81955c00 con 0x558e817fec00 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 25 osd.0 24 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254252+0000 last_rx_front 2019-12-07T19:34:01.753931+0000 -> 2019-12-07T19:34:02.254310+0000 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 25 osd.0 24 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:02.253945+0000) and older pending ping(s) 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002061620s,0.002426586s]) 2019-12-07T19:34:02.253+0000 7fe2e7d3c700 25 osd.0 24 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:01.754054+0000 last_rx_front 2019-12-07T19:34:01.754044+0000 -> 2019-12-07T19:34:02.254333+0000 2019-12-07T19:34:02.253+0000 7fe2e7d3c700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690113426s,18446744073.690501011s]) 2019-12-07T19:34:02.253+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6806/36936 7 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.720448920s delta_ub 0.011811716s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81955c00 con 0x558e817fe000 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6830/36939 7 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.689545969s delta_ub 18446744073.690460381s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81239c00 con 0x558e817fe800 2019-12-07T19:34:02.253+0000 7fe2e7d3c700 25 osd.0 24 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:01.754082+0000 -> 2019-12-07T19:34:02.254354+0000 last_rx_front 2019-12-07T19:34:02.254301+0000 2019-12-07T19:34:02.253+0000 7fe2e7d3c700 25 osd.0 24 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:02.253945+0000) and older pending ping(s) 2019-12-07T19:34:02.253+0000 7fe2e7d3c700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011403109s,0.011811716s]) 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 25 osd.0 24 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:01.754054+0000 -> 2019-12-07T19:34:02.254368+0000 last_rx_front 2019-12-07T19:34:02.254333+0000 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 25 osd.0 24 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:34:02.253945+0000) and older pending ping(s) 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690037689s,18446744073.690460381s]) 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6814/36938 7 ==== osd_ping(ping_reply e24 up_from 13 ping_stamp 2019-12-07T19:34:02.253945+0000/25.708637204s send_stamp 25.703337852s delta_ub 18446744073.704252264s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81239c00 con 0x558e8181a000 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 25 osd.0 24 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:01.754004+0000 -> 2019-12-07T19:34:02.254409+0000 last_rx_front 2019-12-07T19:34:02.254290+0000 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 25 osd.0 24 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:02.253945+0000) and older pending ping(s) 2019-12-07T19:34:02.253+0000 7fe2e6d3a700 20 osd.0 24 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703788851s,18446744073.704252264s]) 2019-12-07T19:34:02.299+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.349+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.399+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.449+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.499+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.550+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.600+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 289548 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.644+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 20 ==== osd_map(24..25 src has 1..25) v4 ==== 1314+0+0 (crc 0 0 0) 0x558e83ff0000 con 0x558e80f39000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 osd.0 24 OSD::ms_dispatch: osd_map(24..25 src has 1..25) v4 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 osd.0 24 do_waiters -- start 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 osd.0 24 do_waiters -- finish 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 osd.0 24 _dispatch 0x558e83ff0000 osd_map(24..25 src has 1..25) v4 2019-12-07T19:34:02.645+0000 7fe2d959f700 3 osd.0 24 handle_osd_map epochs [24,25], i have 24, src has [1,25] 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 osd.0 24 handle_osd_map got inc map for epoch 25 2019-12-07T19:34:02.645+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:32e0e826:::osdmap.24:0# 0x0~0 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1428 size 0x1428 (5160) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8e70 blob([0x102000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d0f730 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0f030 blob([!~1000,0x103000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,428]) SharedBlob(0x558e83d0f0a0 sbid 0x0)) need 0x1000~428 cache has 0x[1000~428] 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:32e0e826:::osdmap.24:0# 0x0~1428 = 5160 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 osd.0 24 add_map_bl 24 5160 bytes 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 osd.0 24 got_full_map 25, nothing requested 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 osd.0 24 handle_osd_map pg_num_history pg_num_history(e25 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps purged_snaps {25={}} 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 osd.0 24 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e25 [1,25] lci=[13,25]) 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e811faf00 seq 17 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c0fb972f:::inc_osdmap.25:0# key 0x7f7fffffffffffffffc0fb97'/!inc_osdmap.25!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:02.645+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c0fb972f:::inc_osdmap.25:0# 0x0~156 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1113 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c0fb972f:::inc_osdmap.25:0# 0x0~156 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~156 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xeaa on front/back, now 0x0~1000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811faf00 1 blobs 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x104000~1000/1000,1000,0 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x104000~1000] 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03a40 blob([0x104000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f960 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.blob(0x558e83d03a40) get_ref 0x0~156 Blob(0x558e83d03a40 blob([0x104000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f960 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.blob(0x558e83d03a40) get_ref init 0x1000, 1000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~156: 0x0~156 Blob(0x558e83d03a40 blob([0x104000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x156) SharedBlob(0x558e83e6f960 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e83e6f978 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x156 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28888) dirty_range mark inline shard dirty 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c0fb972f:::inc_osdmap.25:0# 0x0~156 = 0 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3820e826:::osdmap.25:0# key 0x7f7fffffffffffffff3820e8'&!osdmap.25!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:02.645+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3820e826:::osdmap.25:0# 0x0~1428 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1114 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3820e826:::osdmap.25:0# 0x0~1428 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~428 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbd8 on front/back, now 0x0~1000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811faf00 2 blobs 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x105000~2000/1000,2000,0 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x105000~2000] 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194fdc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194fce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194fdc0 blob([0x105000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194fce0 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.blob(0x558e8194fdc0) get_ref 0x0~1000 Blob(0x558e8194fdc0 blob([0x105000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194fce0 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.blob(0x558e8194fdc0) get_ref init 0x1000, 1000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e8194fdc0 blob([0x105000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194fce0 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e8194fcf8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194fc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194fc70 blob([!~1000,0x106000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5f10 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.blob(0x558e8194fc70) get_ref 0x1000~428 Blob(0x558e8194fc70 blob([!~1000,0x106000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5f10 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.blob(0x558e8194fc70) get_ref init 0x2000, 1000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~428: 0x1000~428 Blob(0x558e8194fc70 blob([!~1000,0x106000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,428]) SharedBlob(0x558e83ac5f10 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e83ac5f28 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1428 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28b08) dirty_range mark inline shard dirty 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3820e826:::osdmap.25:0# 0x0~1428 = 0 2019-12-07T19:34:02.645+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811faf00 0 blobs 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:02.645+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:02.645+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:02.645+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811faf00 0 blobs 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811faf00 cost 22983 (4 ios * 4000 + 6983 bytes) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811faf00 onodes 0x558e80251b80,0x558e817b9180,0x558e83f28780,0x558e83f28a00 shared_blobs 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28888) update #-1:c0fb972f:::inc_osdmap.25:0# 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28888) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c0fb972f:::inc_osdmap.25:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28b08) update #-1:3820e826:::osdmap.25:0# 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28b08) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3820e826:::osdmap.25:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811faf00 allocated 0x[104000~3000] released 0x[] 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 freelist allocate 0x104000~3000 2019-12-07T19:34:02.645+0000 7fe2d959f700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 prepare 2019-12-07T19:34:02.645+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e811faf00 2019-12-07T19:34:02.645+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 aio_wait 2019-12-07T19:34:02.645+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811faf00 2019-12-07T19:34:02.645+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 io_done 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:02.645+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 7628 < min_flush_size 524288 2019-12-07T19:34:02.645+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8800 < min_flush_size 524288 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83f28780 had 1 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83f28a00 had 1 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:02.645+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:34:02.645+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8823 < min_flush_size 524288 2019-12-07T19:34:02.645+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:02.645+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x6e234~2277 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:02.645+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x6e234 0x6e234~2277 to file(ino 16 size 0x70000 mtime 2019-12-07T19:33:58.822165+0000 allocated 70000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~40000]) 2019-12-07T19:34:02.645+0000 7fe2d7a86700 10 bluefs _allocate len 0x4ab from 0 2019-12-07T19:34:02.645+0000 7fe2d7a86700 10 bluefs _allocate len 0x4ab from 1 2019-12-07T19:34:02.645+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd1a0000 2019-12-07T19:34:02.645+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd1a0000~10000/10000,10000,acd1a0000 2019-12-07T19:34:02.645+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x80000 to include allocated 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 29 (was clean) 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x3e234 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x234 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x4ab and padding block with 0xb55 2019-12-07T19:34:02.645+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x80000 2019-12-07T19:34:02.646+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x704ab 2019-12-07T19:34:02.646+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:02.646+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:02.646+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:02.646+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:02.646+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (29) on file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]), flushing log 2019-12-07T19:34:02.646+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:02.646+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.646+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 29 len 0x49 crc 0xaa9a3383) 2019-12-07T19:34:02.646+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x1c000~1000 to file(ino 1 size 0x1c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:02.646+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x1c000 0x1c000~1000 to file(ino 1 size 0x1c000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:02.646+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x1d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:02.646+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1c000 2019-12-07T19:34:02.646+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x1d000 2019-12-07T19:34:02.646+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:02.646+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:34:02.646+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000023 2019-12-07T19:34:02.646+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:02.646+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 29 2019-12-07T19:34:02.646+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.646+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000445278s (0.000005553s flush + 0.000439725s kv commit) 2019-12-07T19:34:02.646+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:02.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:02.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811faf00> 2019-12-07T19:34:02.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:02.646+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 kv_submitted 2019-12-07T19:34:02.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811faf00 2019-12-07T19:34:02.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e811faf00 osr 0x558e8107a500 2019-12-07T19:34:02.646+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) _discard drop 14 0x1b000~1000 2019-12-07T19:34:02.646+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) prepare_write seq 15 0x1b000~1000 crc bc76708a 2019-12-07T19:34:02.646+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) _discard drop 14 0x2000~1000 2019-12-07T19:34:02.646+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) prepare_write seq 15 0x2000~1000 crc a4cd9738 2019-12-07T19:34:02.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:02.646+0000 7fe2d7285700 10 osd.0 24 _committed_osd_maps 25..25 2019-12-07T19:34:02.646+0000 7fe2d7285700 10 osd.0 24 advance to epoch 25 (<= last 25 <= newest_map 25) 2019-12-07T19:34:02.646+0000 7fe2d7285700 7 osd.0 25 consume_map version 25 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 25 identify_splits_and_merges 1.9s3 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 25 identify_splits_and_merges 1.1s1 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 25 identify_splits_and_merges 1.as3 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 25 identify_splits_and_merges 1.2s0 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 25 identify_splits_and_merges 1.bs3 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 25 identify_splits_and_merges 1.4s1 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 25 identify_splits_and_merges 1.ds5 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 25 identify_splits_and_merges 1.es4 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 25 identify_splits_and_merges 1.7s3 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 25 identify_splits_and_merges 1.fs1 e24 to e25 pg_nums {18=16} 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 25 prune_pg_created 2019-12-07T19:34:02.646+0000 7fe2d7285700 10 osd.0:0.consume_map 25 (was 24) 2019-12-07T19:34:02.646+0000 7fe2d7285700 10 osd.0:1.consume_map 25 (was 24) 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:02.646+0000 7fe2d7285700 10 osd.0:2.consume_map 25 (was 24) 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:02.646+0000 7fe2d7285700 10 osd.0:3.consume_map 25 (was 24) 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 2019-12-07T19:34:02.646+0000 7fe2d7285700 10 osd.0:4.consume_map 25 (was 24) 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:02.646+0000 7fe2d7285700 10 osd.0:5.consume_map 25 (was 24) 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:02.646+0000 7fe2d7285700 10 osd.0:6.consume_map 25 (was 24) 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:02.646+0000 7fe2d7285700 10 osd.0:7.consume_map 25 (was 24) 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:02.646+0000 7fe2d7285700 10 osd.0 25 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:02.646+0000 7fe2d7285700 15 osd.0 25 enqueue_peering_evt 1.9s3 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.646+0000 7fe2d7285700 15 osd.0 25 enqueue_peering_evt 1.1s1 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.646+0000 7fe2d7285700 15 osd.0 25 enqueue_peering_evt 1.as3 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.646+0000 7fe2d7285700 15 osd.0 25 enqueue_peering_evt 1.2s0 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.646+0000 7fe2d7285700 15 osd.0 25 enqueue_peering_evt 1.bs3 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.646+0000 7fe2d7285700 15 osd.0 25 enqueue_peering_evt 1.4s1 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.646+0000 7fe2d7285700 15 osd.0 25 enqueue_peering_evt 1.ds5 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.646+0000 7fe2d7285700 15 osd.0 25 enqueue_peering_evt 1.es4 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.646+0000 7fe2d7285700 15 osd.0 25 enqueue_peering_evt 1.7s3 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.646+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.646+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.646+0000 7fe2d7285700 15 osd.0 25 enqueue_peering_evt 1.fs1 epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.646+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.646+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) 2019-12-07T19:34:02.646+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558e819e7000 2019-12-07T19:34:02.646+0000 7fe2d7285700 7 osd.0 25 activate_map version 25 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558e8192b000 2019-12-07T19:34:02.646+0000 7fe2c5261700 10 osd.0 pg_epoch: 24 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.646+0000 7fe2c5261700 20 osd.0:5.update_pg_epoch 1.ds5 24 -> 25 2019-12-07T19:34:02.646+0000 7fe2c5261700 10 osd.0 pg_epoch: 24 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.9s3 24 -> 25 2019-12-07T19:34:02.646+0000 7fe2c5261700 10 osd.0 pg_epoch: 25 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:02.646+0000 7fe2c5261700 10 osd.0 pg_epoch: 25 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.646+0000 7fe2c5261700 20 osd.0 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.646+0000 7fe2c5261700 10 osd.0 pg_epoch: 25 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,3] newacting [1,4,5,0,7,3] 2019-12-07T19:34:02.646+0000 7fe2c5261700 10 osd.0 pg_epoch: 25 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:02.646+0000 7fe2c5261700 20 osd.0 pg_epoch: 25 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 25 2019-12-07T19:34:02.646+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:02.646+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:02.647160+0000 3 0.000056 2019-12-07T19:34:02.646+0000 7fe2c5261700 10 osd.0 pg_epoch: 25 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.646+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:02.646+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:02.646+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:02.647173+0000 0 0.000000 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.646+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558e81a98000 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0:6.update_pg_epoch 1.es4 24 -> 25 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 24 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] handle_advance_map [1,3,4,5,0,2]/[1,3,4,5,0,2] -- 1/1 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,3] newacting [1,4,5,0,7,3] 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] new interval newup [1,3,4,5,0,2] newacting [1,3,4,5,0,2] 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,2147483647]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:02.646+0000 7fe2cb26d700 15 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:02.646+0000 7fe2c4a60700 5 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:02.647247+0000 3 0.000071 2019-12-07T19:34:02.646+0000 7fe2c4a60700 5 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:02.647260+0000 0 0.000000 2019-12-07T19:34:02.646+0000 7fe2c4a60700 5 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(24-24 up [1,4,5,0,7,2147483647](1) acting [1,4,5,0,7,2147483647](1)) up_thru 18 up_from 13 last_epoch_clean 19 interval(24-24 up [1,4,5,0,7,2147483647](1) acting [1,4,5,0,7,2147483647](1)) : primary up 13-18 does not include interval 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=24 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963420426s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] noting past ([18,24] all_participants=0(3),1(0),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4))) 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.963420426s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] new interval newup [1,3,4,5,0,2] newacting [1,3,4,5,0,2] 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.963420426s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.963420426s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2147483647]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.072371497s (mnow 26.102015080s + 9.970356417s) 2019-12-07T19:34:02.646+0000 7fe2c4a60700 15 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,24)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:02.646+0000 7fe2cb26d700 1 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] start_peering_interval up [1,4,5,0,7,2147483647] -> [1,4,5,0,7,3], acting [1,4,5,0,7,2147483647] -> [1,4,5,0,7,3], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 3 -> 3, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(24-24 up [1,3,4,5,0,2147483647](1) acting [1,3,4,5,0,2147483647](1)) up_thru 18 up_from 13 last_epoch_clean 19 interval(24-24 up [1,3,4,5,0,2147483647](1) acting [1,3,4,5,0,2147483647](1)) : primary up 13-18 does not include interval 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963364774s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] noting past ([18,24] all_participants=0(4),1(0),3(1),4(2),5(3),6(5) intervals=([18,23] acting 0(4),1(0),3(1),4(2),5(3),6(5))) 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] release_backoffs [1:90000000::::0,1:a0000000::::head) 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.963364774s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.963364774s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.963364774s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.072387811s (mnow 26.102083049s + 9.970304762s) 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:02.646+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.646+0000 7fe2c4a60700 1 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] start_peering_interval up [1,3,4,5,0,2147483647] -> [1,3,4,5,0,2], acting [1,3,4,5,0,2147483647] -> [1,3,4,5,0,2], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:02.646+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.646+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:02.646+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558e81938000 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] release_backoffs [1:70000000::::0,1:80000000::::head) 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:02.646+0000 7fe2c5a62700 10 osd.0 pg_epoch: 24 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.646+0000 7fe2c5a62700 20 osd.0:4.update_pg_epoch 1.4s1 24 -> 25 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.646+0000 7fe2c5a62700 10 osd.0 pg_epoch: 24 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970356417s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:02.646+0000 7fe2c5a62700 10 osd.0 pg_epoch: 25 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:02.646+0000 7fe2c5a62700 10 osd.0 pg_epoch: 25 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.646+0000 7fe2c5a62700 20 osd.0 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.646+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970204087s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:02.647482+0000 1 0.000323 2019-12-07T19:34:02.646+0000 7fe2c5a62700 10 osd.0 pg_epoch: 25 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:02.646+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970204087s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:02.646+0000 7fe2c5a62700 10 osd.0 pg_epoch: 25 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:02.646+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970204087s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:02.646+0000 7fe2c5a62700 20 osd.0 pg_epoch: 25 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 25 2019-12-07T19:34:02.646+0000 7fe2cb26d700 1 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970204087s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.646+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.646+0000 7fe2c5a62700 10 osd.0 pg_epoch: 25 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.646+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:02.646+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970204087s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:02.647514+0000 0 0.000000 2019-12-07T19:34:02.646+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.646+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970204087s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:02.646+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970204087s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 25 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970304762s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.646+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.646+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.646+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.646+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558e8193b000 2019-12-07T19:34:02.646+0000 7fe2c4a60700 5 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970133255s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:02.647569+0000 1 0.000291 2019-12-07T19:34:02.646+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.646+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.as3 24 -> 25 2019-12-07T19:34:02.646+0000 7fe2c4a60700 5 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970133255s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:02.646+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] handle_advance_map [1,4,2,0,5,3]/[1,4,2,0,5,3] -- 1/1 2019-12-07T19:34:02.646+0000 7fe2c4a60700 5 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970133255s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:02.646+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.646+0000 7fe2c4a60700 1 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970133255s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:02.646+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 22 ==== pg_notify2(1.2s0 1.2s0 (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.972591842s) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e25/25) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e8122b800 con 0x558e818d6800 2019-12-07T19:34:02.646+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] new interval newup [1,4,2,0,5,3] newacting [1,4,2,0,5,3] 2019-12-07T19:34:02.646+0000 7fe2c4a60700 5 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970133255s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:02.647611+0000 0 0.000000 2019-12-07T19:34:02.646+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:02.646+0000 7fe2c4a60700 5 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970133255s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:02.646+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:02.647622+0000 3 0.000063 2019-12-07T19:34:02.646+0000 7fe2c4a60700 20 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970133255s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 25 2019-12-07T19:34:02.646+0000 7fe2e753b700 15 osd.0 25 enqueue_peering_evt 1.2s0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(4) notify: (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.972591842s) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.646+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(4) notify: (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.972591842s) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.646+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:02.647631+0000 0 0.000000 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:02.646+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:02.646+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970133255s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.646+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:02.646+0000 7fe2c4a60700 10 osd.0 25 send_incremental_map 24 -> 25 to 0x558e818d7c00 v2:172.21.15.131:6823/37078 2019-12-07T19:34:02.646+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:02.646+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:02.646+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] new interval newup [1,4,2,0,5,3] newacting [1,4,2,0,5,3] 2019-12-07T19:34:02.646+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.646+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.646+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c0fb972f:::inc_osdmap.25:0# 0x0~0 2019-12-07T19:34:02.646+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558e81928000 2019-12-07T19:34:02.646+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:02.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~156 size 0x156 (342) 2019-12-07T19:34:02.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,2147483647,2,0,5,4]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03a40 blob([0x104000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x156) SharedBlob(0x558e83e6f960 sbid 0x0)) need 0x0~156 cache has 0x[0~156] 2019-12-07T19:34:02.647+0000 7fe2c6a64700 15 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:02.647+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c0fb972f:::inc_osdmap.25:0# 0x0~156 = 342 2019-12-07T19:34:02.647+0000 7fe2c4a60700 10 osd.0 25 add_map_inc_bl 25 342 bytes 2019-12-07T19:34:02.647+0000 7fe2c4a60700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(25..25 src has 1..25) v4 -- 0x558e817de480 con 0x558e818d7c00 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(24-24 up [1,2147483647,2,0,5,4](1) acting [1,2147483647,2,0,5,4](1)) up_thru 18 up_from 13 last_epoch_clean 19 interval(24-24 up [1,2147483647,2,0,5,4](1) acting [1,2147483647,2,0,5,4](1)) : primary up 13-18 does not include interval 2019-12-07T19:34:02.647+0000 7fe2c4a60700 10 osd.0 25 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 24 -> 25 (shared) 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964343587s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] noting past ([18,24] all_participants=0(3),1(0),2(2),4(5),5(4),6(1) intervals=([18,23] acting 0(3),1(0),2(2),4(5),5(4),6(1))) 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.964343587s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.964343587s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:02.647+0000 7fe2c4a60700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.es0 1.es0 (query:25 sent:25 1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970133255s) 4->0 ([18,24] all_participants=0(4),1(0),3(1),4(2),5(3),6(5) intervals=([18,23] acting 0(4),1(0),3(1),4(2),5(3),6(5)))) e25/25) v1 -- 0x558e83ebca00 con 0x558e818d7c00 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.964343587s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.073166239s (mnow 26.102451143s + 9.970715096s) 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:02.647+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:02.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e83c56300 seq 15 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:02.647+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:02.647+0000 7fe2c6a64700 1 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] start_peering_interval up [1,2147483647,2,0,5,4] -> [1,4,2,0,5,3], acting [1,2147483647,2,0,5,4] -> [1,4,2,0,5,3], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 3 -> 3, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:02.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:02.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:02.647+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 23 ==== osd_map(25..25 src has 1..25) v4 ==== 386+0+0 (unknown 0 0 0) 0x558e83f2afc0 con 0x558e818d6800 2019-12-07T19:34:02.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:02.647+0000 7fe2d4a80700 20 osd.0 25 OSD::ms_dispatch: osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.647+0000 7fe2d4a80700 10 osd.0 25 do_waiters -- start 2019-12-07T19:34:02.647+0000 7fe2d4a80700 10 osd.0 25 do_waiters -- finish 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] release_backoffs [1:50000000::::0,1:60000000::::head) 2019-12-07T19:34:02.647+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:02.647+0000 7fe2d4a80700 20 osd.0 25 _dispatch 0x558e83f2afc0 osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.647+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56300 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:02.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56300 onodes shared_blobs 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:02.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56300 allocated 0x[] released 0x[] 2019-12-07T19:34:02.647+0000 7fe2d4a80700 3 osd.0 25 handle_osd_map epochs [25,25], i have 25, src has [1,25] 2019-12-07T19:34:02.647+0000 7fe2d4a80700 10 osd.0 25 no new maps here, dropping 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:02.647+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 prepare 2019-12-07T19:34:02.647+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56300 2019-12-07T19:34:02.647+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 io_done 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:02.647+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs3 24 -> 25 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 24 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558e8122e000 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] new interval newup [4,0,2,7,1,5] newacting [4,0,2,7,1,5] 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.7s3 24 -> 25 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.647+0000 7fe2ca26b700 5 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:02.647871+0000 3 0.000075 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.647+0000 7fe2ca26b700 5 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:02.647881+0000 0 0.000000 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970715096s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.647+0000 7fe2ca26b700 5 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] new interval newup [1,4,3,0,7,2] newacting [1,4,3,0,7,2] 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=24 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.647+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970584258s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:02.647897+0000 1 0.000274 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=25 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:02.647+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970584258s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:02.647+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970584258s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=25 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:02.647+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:02.647908+0000 3 0.000057 2019-12-07T19:34:02.647+0000 7fe2c6a64700 1 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970584258s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=25 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:02.647+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:02.647921+0000 0 0.000000 2019-12-07T19:34:02.647+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970584258s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:02.647925+0000 0 0.000000 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=25 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] new interval newup [4,0,2,7,1,5] newacting [4,0,2,7,1,5] 2019-12-07T19:34:02.647+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:02.647+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970584258s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=25 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=24 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,2147483647,2,0,1,5]p4(0) r=3 lpr=25 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970584258s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 25 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:02.647+0000 7fe2ca26b700 15 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,0,2,7,1,5]p4(0) r=3 lpr=25 pi=[18,24)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970584258s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] new interval newup [1,4,3,0,7,2] newacting [1,4,3,0,7,2] 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(24-24 up [4,2147483647,2,0,1,5](4) acting [4,2147483647,2,0,1,5](4)) up_thru 18 up_from 13 last_epoch_clean 19 interval(24-24 up [4,2147483647,2,0,1,5](4) acting [4,2147483647,2,0,1,5](4)) : primary up 13-18 does not include interval 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:02.647+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.as0 1.as0 (query:25 sent:25 1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970584258s) 3->0 ([18,24] all_participants=0(3),1(0),2(2),4(5),5(4),6(1) intervals=([18,23] acting 0(3),1(0),2(2),4(5),5(4),6(1)))) e25/25) v1 -- 0x558e80ff3800 con 0x558e818d7c00 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.961850928s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] noting past ([18,24] all_participants=0(3),1(4),2(2),4(0),5(5),6(1) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,2147483647,4,0,7,2]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.961850928s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:02.647+0000 7fe2c425f700 15 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,24)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.961850928s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e81a0b500 seq 11 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.961850928s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] init_hb_stamps now [] 2019-12-07T19:34:02.647+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.069286165s (mnow 26.102697308s + 9.966588857s) 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(24-24 up [1,2147483647,4,0,7,2](1) acting [1,2147483647,4,0,7,2](1)) up_thru 18 up_from 13 last_epoch_clean 19 interval(24-24 up [1,2147483647,4,0,7,2](1) acting [1,2147483647,4,0,7,2](1)) : primary up 13-18 does not include interval 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b500 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b500 onodes shared_blobs 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b500 allocated 0x[] released 0x[] 2019-12-07T19:34:02.647+0000 7fe2ca26b700 1 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] start_peering_interval up [4,2147483647,2,0,1,5] -> [4,0,2,7,1,5], acting [4,2147483647,2,0,1,5] -> [4,0,2,7,1,5], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role 3 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.965761400s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] noting past ([18,24] all_participants=0(3),1(0),2(5),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4))) 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 prepare 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b500 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 io_done 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.965761400s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] release_backoffs [1:d0000000::::0,1:e0000000::::head) 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.965761400s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558e81075000 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.965761400s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.073818636s (mnow 26.102749403s + 9.971069233s) 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.2s0 24 -> 25 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 24 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:02.647+0000 7fe2c425f700 1 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] start_peering_interval up [1,2147483647,4,0,7,2] -> [1,4,3,0,7,2], acting [1,2147483647,4,0,7,2] -> [1,4,3,0,7,2], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 3 -> 3, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Peering advmap 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] adjust_need_up_thru now 24, need_up_thru now false 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] release_backoffs [1:e0000000::::0,1:f0000000::::head) 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] new interval newup [0,4,1,5,7,3] newacting [0,4,1,5,7,3] 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] plpg_on_role_change 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:02.647+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] exit Started/Primary/Peering/WaitUpThru 2019-12-07T19:34:02.648121+0000 6 0.000620 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 osd.0 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Leaving Peering 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966588857s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.647+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] exit Started/Primary/Peering 2019-12-07T19:34:02.648138+0000 0 0.000000 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:02.647+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] exit Started/Primary 2019-12-07T19:34:02.648147+0000 0 0.000000 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.647+0000 7fe2ca26b700 5 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966450134s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:02.648149+0000 1 0.000279 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:02.647+0000 7fe2ca26b700 5 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966450134s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:02.647+0000 7fe2ca26b700 5 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966450134s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.647+0000 7fe2ca26b700 1 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966450134s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:02.647+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:02.648169+0000 0 0.000000 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.647+0000 7fe2ca26b700 5 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966450134s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:02.648176+0000 0 0.000000 2019-12-07T19:34:02.647+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=24 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.647+0000 7fe2ca26b700 5 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966450134s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971069233s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=25 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966450134s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 25 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=25 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:02.647+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970933308s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:02.648200+0000 1 0.000292 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=25 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:02.647+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970933308s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=25 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] new interval newup [0,4,1,5,7,3] newacting [0,4,1,5,7,3] 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966450134s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=25 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:02.647+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970933308s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 25 send_incremental_map 24 -> 25 to 0x558e818fe400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,2147483647,5,3]p0(0) r=0 lpr=25 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 25 clear_ready_to_merge 1.2s0 2019-12-07T19:34:02.647+0000 7fe2c425f700 1 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970933308s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:02.647+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(25..25 src has 1..25) v4 -- 0x558e83fd0fc0 con 0x558e818fe400 2019-12-07T19:34:02.647+0000 7fe2c6a64700 15 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,24)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:02.647+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970933308s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:02.648231+0000 0 0.000000 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 osd.0 25 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 24 -> 25 (shared) 2019-12-07T19:34:02.647+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970933308s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970933308s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 25 2019-12-07T19:34:02.647+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:25 sent:25 1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966450134s) 3->0 ([18,24] all_participants=0(3),1(4),2(2),4(0),5(5),6(1) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e25/25) v1 -- 0x558e83ebd800 con 0x558e818fe400 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(24-24 up [0,4,1,2147483647,5,3](0) acting [0,4,1,2147483647,5,3](0)) up_thru 18 up_from 13 last_epoch_clean 19 interval(24-24 up [0,4,1,2147483647,5,3](0) acting [0,4,1,2147483647,5,3](0)) : primary up 13-18 does not include interval 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e8106c000 seq 25 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] noting past ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970933308s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:02.647+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:25 sent:25 1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970933308s) 3->0 ([18,24] all_participants=0(3),1(0),2(5),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e25/25) v1 -- 0x558e8404ae00 con 0x558e818d7c00 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d0af0,0x558e811d01c0,0x558e811d18f0,0x558e811d1b20,0x558e811d1570] 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 26.102991068s + 0s) 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e83d4d200 seq 24 2019-12-07T19:34:02.647+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:02.647+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:24.571771+0000, must = 0 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:02.647+0000 7fe2c6a64700 1 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] start_peering_interval up [0,4,1,2147483647,5,3] -> [0,4,1,5,7,3], acting [0,4,1,2147483647,5,3] -> [0,4,1,5,7,3], acting_primary 0(0) -> 0, up_primary 0(0) -> 0, role 0 -> 0, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c000 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c000 onodes shared_blobs 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c000 allocated 0x[] released 0x[] 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 prepare 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c000 2019-12-07T19:34:02.647+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 io_done 2019-12-07T19:34:02.647+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:02.647+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4d200 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4d200 onodes shared_blobs 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4d200 allocated 0x[] released 0x[] 2019-12-07T19:34:02.647+0000 7fe2cc26f700 20 osd.0 25 agent_entry empty queue 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d200 prepare 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4d200 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d200 io_done 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558e81a9b000 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.fs1 24 -> 25 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 24 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] handle_advance_map [3,0,5,4,1,2]/[3,0,5,4,1,2] -- 3/3 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.647+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] new interval newup [3,0,5,4,1,2] newacting [3,0,5,4,1,2] 2019-12-07T19:34:02.647+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:02.648+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:02.648+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:02.648716+0000 3 0.000068 2019-12-07T19:34:02.648+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(4) notify: (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.972591842s) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.648+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.647+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:02.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:02.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:02.646+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:02.648+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970204087s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.648+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:02.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:02.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2538 < min_flush_size 524288 2019-12-07T19:34:02.647+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:02.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:02.648+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:25 sent:25 1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970204087s) 3->0 ([18,24] all_participants=0(3),1(0),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4)))) e25/25) v1 -- 0x558e83c9c700 con 0x558e818d7c00 2019-12-07T19:34:02.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3807 < min_flush_size 524288 2019-12-07T19:34:02.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:34:02.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5076 < min_flush_size 524288 2019-12-07T19:34:02.648+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:02.648+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83d4cc00 seq 15 2019-12-07T19:34:02.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:02.648+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:02.648+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:02.648+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:02.648+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:02.648+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:02.648+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 55 ==== pg_notify2(1.2s0 1.2s0 (query:25 sent:25 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971150263s) 1->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e25/25) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e83ebd100 con 0x558e818fe400 2019-12-07T19:34:02.648+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4cc00 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:02.648+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4cc00 onodes shared_blobs 2019-12-07T19:34:02.648+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4cc00 allocated 0x[] released 0x[] 2019-12-07T19:34:02.648+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 prepare 2019-12-07T19:34:02.648+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4cc00 2019-12-07T19:34:02.648+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 io_done 2019-12-07T19:34:02.648+0000 7fe2e6d3a700 15 osd.0 25 enqueue_peering_evt 1.2s0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 4(1) notify: (query:25 sent:25 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971150263s) 1->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.648+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 4(1) notify: (query:25 sent:25 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971150263s) 1->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.648+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.648+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 NullEvt) prio 255 cost 10 e25) pg 0x558e819e4000 2019-12-07T19:34:02.648+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 56 ==== pg_query2(1.bs1 1.bs1 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e83b09680 con 0x558e818fe400 2019-12-07T19:34:02.648+0000 7fe2e6d3a700 15 osd.0 25 enqueue_peering_evt 1.bs1 epoch_sent: 25 epoch_requested: 25 MQuery 1.bs1 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.648+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] handle_advance_map: 25 2019-12-07T19:34:02.648+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.bs1 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.648+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.1s1 24 -> 25 2019-12-07T19:34:02.648+0000 7fe2cb26d700 10 osd.0 pg_epoch: 24 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] handle_advance_map [3,0,2,4,5,1]/[3,0,2,4,5,1] -- 3/3 2019-12-07T19:34:02.648+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 57 ==== pg_query2(1.bs3 1.bs3 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e83b09680 con 0x558e818fe400 2019-12-07T19:34:02.648+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 101 ==== pg_query2(1.8s5 1.8s5 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8402fb80 con 0x558e818d7c00 2019-12-07T19:34:02.648+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:02.648+0000 7fe2e7d3c700 15 osd.0 25 enqueue_peering_evt 1.8s5 epoch_sent: 25 epoch_requested: 25 MQuery 1.8s5 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.648+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.648+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.8s5 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.648+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] new interval newup [3,0,2,4,5,1] newacting [3,0,2,4,5,1] 2019-12-07T19:34:02.648+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:02.648+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 102 ==== pg_query2(1.9s3 1.9s3 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8402fb80 con 0x558e818d7c00 2019-12-07T19:34:02.648+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:02.649036+0000 3 0.000065 2019-12-07T19:34:02.648+0000 7fe2e7d3c700 15 osd.0 25 enqueue_peering_evt 1.9s3 epoch_sent: 25 epoch_requested: 25 MQuery 1.9s3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.648+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:02.648732+0000 0 0.000000 2019-12-07T19:34:02.648+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:02.649045+0000 0 0.000000 2019-12-07T19:34:02.648+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.9s3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.648+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:02.648+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:02.648+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.648+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=24 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.648+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:02.648+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:02.648+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:02.648+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] new interval newup [3,0,5,4,1,2] newacting [3,0,5,4,1,2] 2019-12-07T19:34:02.648+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:02.648+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.8s5 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.648+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2147483647]p3(0) r=1 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.648+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.8s5 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0 2019-12-07T19:34:02.648+0000 7fe2e6d3a700 15 osd.0 25 enqueue_peering_evt 1.bs3 epoch_sent: 25 epoch_requested: 25 MQuery 1.bs3 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.648+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.bs3 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.648+0000 7fe2c425f700 15 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,24)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:02.648+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(24-24 up [3,0,5,4,1,2147483647](3) acting [3,0,5,4,1,2147483647](3)) up_thru 18 up_from 13 last_epoch_clean 19 interval(24-24 up [3,0,5,4,1,2147483647](3) acting [3,0,5,4,1,2147483647](3)) : primary up 13-18 does not include interval 2019-12-07T19:34:02.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5099 < min_flush_size 524288 2019-12-07T19:34:02.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5099 < min_flush_size 524288 2019-12-07T19:34:02.648+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.963854524s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] noting past ([18,24] all_participants=0(1),1(4),3(0),4(3),5(2),6(5) intervals=([18,23] acting 0(1),1(4),3(0),4(3),5(2),6(5))) 2019-12-07T19:34:02.648+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x704ab~13eb to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.648+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x704ab 0x704ab~13eb to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.648+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.9s3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.648+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.9s3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0x558e8192b000 2019-12-07T19:34:02.648+0000 7fe2c7265700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970204087s) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MQuery 1.9s3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.648+0000 7fe2c7265700 20 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.072371497s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 2019-12-07T19:34:02.648+0000 7fe2c7265700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:02.648+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:02.648+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:25 sent:25 1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 3->0 ([18,24] all_participants=0(3),1(0),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4)))) e25/25) v1 -- 0x558e81228000 con 0x558e818d7c00 2019-12-07T19:34:02.648+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:02.648+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e8417a000 seq 16 2019-12-07T19:34:02.648+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:02.648+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:02.648+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:02.648+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417a000 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:02.648+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417a000 onodes shared_blobs 2019-12-07T19:34:02.648+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417a000 allocated 0x[] released 0x[] 2019-12-07T19:34:02.648+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a000 prepare 2019-12-07T19:34:02.648+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417a000 2019-12-07T19:34:02.648+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a000 io_done 2019-12-07T19:34:02.648+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.648+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 103 ==== osd_map(25..25 src has 1..25) v4 ==== 386+0+0 (unknown 0 0 0) 0x558e817de480 con 0x558e818d7c00 2019-12-07T19:34:02.648+0000 7fe2d4a80700 20 osd.0 25 OSD::ms_dispatch: osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.648+0000 7fe2d4a80700 10 osd.0 25 do_waiters -- start 2019-12-07T19:34:02.648+0000 7fe2d4a80700 10 osd.0 25 do_waiters -- finish 2019-12-07T19:34:02.648+0000 7fe2d4a80700 20 osd.0 25 _dispatch 0x558e817de480 osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.648+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.bs1 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.648+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.648+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.bs1 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0 2019-12-07T19:34:02.648+0000 7fe2ca26b700 10 osd.0 25 handle_pg_query_nopg 1.bs1 2019-12-07T19:34:02.648+0000 7fe2ca26b700 10 osd.0 25 pg 1.bs1 dne 2019-12-07T19:34:02.648+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify((query:25 sent:25 1.bs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 25) v7 -- 0x558e8403eb40 con 0x558e818fe400 2019-12-07T19:34:02.648+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:02.647+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.648+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] requeue_op 0x558e819438c0 2019-12-07T19:34:02.648+0000 7fe2c6a64700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e25) shuffled w/ OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(4) notify: (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.972591842s) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.648+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] requeue_op 0x558e81943080 2019-12-07T19:34:02.648+0000 7fe2c6a64700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e25) shuffled w/ OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e25) 2019-12-07T19:34:02.648+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] requeue_op 0x558e819431e0 2019-12-07T19:34:02.648+0000 7fe2c6a64700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e25) shuffled w/ OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e25) 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:02.648+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:02.648+0000 7fe2c6a64700 15 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] publish_stats_to_osd 24:35 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.648+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:02.648+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] [0,4,1,2147483647,5,3] -> [0,4,1,5,7,3], replicas changed 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.648+0000 7fe2c6a64700 20 osd.0 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5 -> 0,1,3,4,5,7 2019-12-07T19:34:02.648+0000 7fe2cba6e700 10 osd.0 25 handle_pg_query_nopg 1.8s5 2019-12-07T19:34:02.648+0000 7fe2c6a64700 20 osd.0 25 need_heartbeat_peer_update 2019-12-07T19:34:02.648+0000 7fe2cba6e700 10 osd.0 25 pg 1.8s5 dne 2019-12-07T19:34:02.648+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:02.649522+0000 1 0.001410 2019-12-07T19:34:02.648+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:02.648+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:25 sent:25 1.8s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 25) v7 -- 0x558e8403e960 con 0x558e818d7c00 2019-12-07T19:34:02.648+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:02.648+0000 7fe2c6a64700 1 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] state: transitioning to Primary 2019-12-07T19:34:02.648+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:02.648+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:02.649550+0000 0 0.000000 2019-12-07T19:34:02.648+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] enter Started/Primary 2019-12-07T19:34:02.648+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] enter Started/Primary/Peering 2019-12-07T19:34:02.648+0000 7fe2c6a64700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] enter Started/Primary/Peering/GetInfo 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] build_prior all_probe 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] build_prior maybe_rw interval:18, acting: 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] build_prior prior osd.6 is down 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] build_prior final: probe 0(0),1(2),3(5),4(1),5(3),5(4),7(4) down 6 blocked_by {} 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] up_thru 24 < same_since 25, must notify monitor 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: querying info from osd.1(2) 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: querying info from osd.3(5) 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: querying info from osd.4(1) 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: querying info from osd.5(3) 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: querying info from osd.5(4) 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: querying info from osd.7(4) 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] check_prior_readable_down_osds prior_readable_down_osds osd.6 is dead as of epoch 25 2019-12-07T19:34:02.648+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] empty prior_readable_down_osds, clearing ub 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:02.649+0000 7fe2c6a64700 15 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] publish_stats_to_osd 24:36 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 25 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.2s2 1.2s2 query(info 0'0 epoch_sent 25) e25/25) v1 -- 0x558e8402f400 con 0x558e818d7c00 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 send_incremental_map 24 -> 25 to 0x558e818d7000 v2:172.21.15.131:6812/37076 2019-12-07T19:34:02.649+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(25..25 src has 1..25) v4 -- 0x558e83f2a6c0 con 0x558e818d7000 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 24 -> 25 (shared) 2019-12-07T19:34:02.649+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_query2(1.2s5 1.2s5 query(info 0'0 epoch_sent 25) e25/25) v1 -- 0x558e8417c000 con 0x558e818d7000 2019-12-07T19:34:02.649+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_query2(1.2s1 1.2s1 query(info 0'0 epoch_sent 25) e25/25) v1 -- 0x558e8417c280 con 0x558e818fe400 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 send_incremental_map 24 -> 25 to 0x558e818d6800 v2:172.21.15.109:6802/36936 2019-12-07T19:34:02.649+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(25..25 src has 1..25) v4 -- 0x558e83dd06c0 con 0x558e818d6800 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 24 -> 25 (shared) 2019-12-07T19:34:02.649+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.2s3 1.2s3 query(info 0'0 epoch_sent 25) e25/25) v1 -- 0x558e8417c500 con 0x558e818d6800 2019-12-07T19:34:02.649+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 25) e25/25) v1 -- 0x558e8417c780 con 0x558e818d6800 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 send_incremental_map 24 -> 25 to 0x558e818fe000 v2:172.21.15.109:6826/36939 2019-12-07T19:34:02.649+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(25..25 src has 1..25) v4 -- 0x558e83dd0900 con 0x558e818fe000 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 24 -> 25 (shared) 2019-12-07T19:34:02.649+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 25) e25/25) v1 -- 0x558e8417ca00 con 0x558e818fe000 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e81a0bb00 seq 22 2019-12-07T19:34:02.649+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0bb00 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0bb00 onodes shared_blobs 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 prepare 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0bb00 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 io_done 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 queue_want_up_thru now 25 (was 24), currently 24 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 send_alive up_thru currently 24 want 25 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 send_alive want 25 2019-12-07T19:34:02.649+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- osd_alive(want up_thru 25 have 25) v1 -- 0x558e819b1860 con 0x558e80f39000 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(4) notify: (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.972591842s) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e25) pg 0x558e81075000 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 dequeue_op 0x558e819431e0 prio 63 cost 4 latency 0.995587 osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 send_incremental_map 24 -> 25 to 0x558e83d68000 172.21.15.131:0/224485236 2019-12-07T19:34:02.649+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_map(25..25 src has 1..25) v4 -- 0x558e83dd1200 con 0x558e83d68000 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 maybe_share_map con 0x558e83d68000 172.21.15.131:0/224485236 map epoch 24 -> 25 (shared) 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 dequeue_op 0x558e819431e0 finish 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 4(1) notify: (query:25 sent:25 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971150263s) 1->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 4(1) notify: (query:25 sent:25 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971150263s) 1->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(4) notify: (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.972591842s) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558e81075000 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(4) notify: (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.972591842s) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.5(4) 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.972591842s) 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:07.707407+0000, must = 0 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] osd.5(4) has stray content: 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.972591842s) 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 queue_want_up_thru want 25 <= queued 25, currently 24 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e25) queued 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25),OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 4(1) notify: (query:25 sent:25 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971150263s) 1->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558e81075000 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 4(1) notify: (query:25 sent:25 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971150263s) 1->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.4(1) 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971150263s) 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:24.204934+0000, must = 0 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 4 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 queue_want_up_thru want 25 <= queued 25, currently 24 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e25) queued 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e25) pg 0x558e81075000 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 dequeue_op 0x558e81943080 prio 63 cost 4 latency 0.995676 osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] 2019-12-07T19:34:02.649+0000 7fe2c6a64700 10 osd.0 25 dequeue_op 0x558e81943080 finish 2019-12-07T19:34:02.649+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.bs3 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.bs3 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0x558e81928000 2019-12-07T19:34:02.649+0000 7fe2c6263700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966450134s) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MQuery 1.bs3 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.649+0000 7fe2c6263700 20 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY pruub 36.069286165s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 2019-12-07T19:34:02.649+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 104 ==== pg_query2(1.7s3 1.7s3 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8402fb80 con 0x558e818d7c00 2019-12-07T19:34:02.649+0000 7fe2c6263700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:02.649+0000 7fe2e7d3c700 15 osd.0 25 enqueue_peering_evt 1.7s3 epoch_sent: 25 epoch_requested: 25 MQuery 1.7s3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.649+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.7s3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 105 ==== pg_query2(1.es4 1.es4 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8402fb80 con 0x558e818d7c00 2019-12-07T19:34:02.649+0000 7fe2e7d3c700 15 osd.0 25 enqueue_peering_evt 1.es4 epoch_sent: 25 epoch_requested: 25 MQuery 1.es4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.649+0000 7fe2c6263700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:25 sent:25 1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 3->0 ([18,24] all_participants=0(3),1(4),2(2),4(0),5(5),6(1) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e25/25) v1 -- 0x558e80ff3100 con 0x558e818fe400 2019-12-07T19:34:02.649+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.es4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.649+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:34:02.649+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e8195e300 seq 26 2019-12-07T19:34:02.649+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:02.649+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:02.649+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:02.649+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e300 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:02.649+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e300 onodes shared_blobs 2019-12-07T19:34:02.649+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e300 allocated 0x[] released 0x[] 2019-12-07T19:34:02.649+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 prepare 2019-12-07T19:34:02.649+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e300 2019-12-07T19:34:02.649+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 io_done 2019-12-07T19:34:02.649+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.7s3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.649+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.649+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.7s3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0x558e8122e000 2019-12-07T19:34:02.649+0000 7fe2c8267700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970933308s) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MQuery 1.7s3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.649+0000 7fe2c8267700 20 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY pruub 36.073818636s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 2019-12-07T19:34:02.649+0000 7fe2c8267700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:02.649+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:02.649+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:25 sent:25 1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 3->0 ([18,24] all_participants=0(3),1(0),2(5),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e25/25) v1 -- 0x558e8404b500 con 0x558e818d7c00 2019-12-07T19:34:02.649+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:02.649+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e83d4d500 seq 25 2019-12-07T19:34:02.649+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:02.649+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:02.649+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:02.649+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4d500 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:02.649+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4d500 onodes shared_blobs 2019-12-07T19:34:02.649+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4d500 allocated 0x[] released 0x[] 2019-12-07T19:34:02.649+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 prepare 2019-12-07T19:34:02.649+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4d500 2019-12-07T19:34:02.649+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 io_done 2019-12-07T19:34:02.649+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.649+0000 7fe2d4a80700 3 osd.0 25 handle_osd_map epochs [25,25], i have 25, src has [1,25] 2019-12-07T19:34:02.649+0000 7fe2d4a80700 10 osd.0 25 no new maps here, dropping 2019-12-07T19:34:02.649+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 58 ==== osd_map(25..25 src has 1..25) v4 ==== 386+0+0 (unknown 0 0 0) 0x558e8123afc0 con 0x558e818fe400 2019-12-07T19:34:02.649+0000 7fe2d4a80700 20 osd.0 25 OSD::ms_dispatch: osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.649+0000 7fe2d4a80700 10 osd.0 25 do_waiters -- start 2019-12-07T19:34:02.649+0000 7fe2d4a80700 10 osd.0 25 do_waiters -- finish 2019-12-07T19:34:02.649+0000 7fe2d4a80700 20 osd.0 25 _dispatch 0x558e8123afc0 osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.649+0000 7fe2d4a80700 3 osd.0 25 handle_osd_map epochs [25,25], i have 25, src has [1,25] 2019-12-07T19:34:02.649+0000 7fe2d4a80700 10 osd.0 25 no new maps here, dropping 2019-12-07T19:34:02.648+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.963854524s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:02.649+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.963854524s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.963854524s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d1570] 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.071369293s (mnow 26.105182210s + 9.966187083s) 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:02.649+0000 7fe2c425f700 1 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] start_peering_interval up [3,0,5,4,1,2147483647] -> [3,0,5,4,1,2], acting [3,0,5,4,1,2147483647] -> [3,0,5,4,1,2], acting_primary 3(0) -> 3, up_primary 3(0) -> 3, role 1 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] release_backoffs [1:f0000000::::0,MAX) 2019-12-07T19:34:02.649+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.649+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:02.649+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.649+0000 7fe2c425f700 20 osd.0 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966187083s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.649+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966054638s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:02.650629+0000 1 0.002172 2019-12-07T19:34:02.649+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966054638s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:02.649+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966054638s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:02.649+0000 7fe2c425f700 1 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966054638s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:02.649+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966054638s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:02.650657+0000 0 0.000000 2019-12-07T19:34:02.649+0000 7fe2c425f700 5 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966054638s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:02.649+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966054638s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 25 2019-12-07T19:34:02.649+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:02.650+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966054638s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.650+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:02.650+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.fs0 1.fs0 (query:25 sent:25 1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966054638s) 1->0 ([18,24] all_participants=0(1),1(4),3(0),4(3),5(2),6(5) intervals=([18,23] acting 0(1),1(4),3(0),4(3),5(2),6(5)))) e25/25) v1 -- 0x558e8122b100 con 0x558e818d7000 2019-12-07T19:34:02.650+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:02.650+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e83d4c900 seq 19 2019-12-07T19:34:02.650+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:02.650+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:02.650+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:34:02.650+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:02.650+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:02.650+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c900 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:02.650+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c900 onodes shared_blobs 2019-12-07T19:34:02.650+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c900 allocated 0x[] released 0x[] 2019-12-07T19:34:02.650+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 prepare 2019-12-07T19:34:02.650+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c900 2019-12-07T19:34:02.650+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 io_done 2019-12-07T19:34:02.650+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.650+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.650+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 13 ==== pg_notify((query:25 sent:25 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 25) v7 ==== 985+0+0 (unknown 0 0 0) 0x558e817b43c0 con 0x558e818fe000 2019-12-07T19:34:02.650+0000 7fe2e753b700 7 osd.0 25 handle_fast_pg_notify pg_notify((query:25 sent:25 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 25) v7 from osd.7 2019-12-07T19:34:02.650+0000 7fe2e753b700 15 osd.0 25 enqueue_peering_evt 1.2s0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 7(4) notify: (query:25 sent:25 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.650+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 7(4) notify: (query:25 sent:25 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.650+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 24 ==== pg_notify((query:25 sent:25 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 25) v7 ==== 985+0+0 (unknown 0 0 0) 0x558e817b43c0 con 0x558e818d6800 2019-12-07T19:34:02.650+0000 7fe2e753b700 7 osd.0 25 handle_fast_pg_notify pg_notify((query:25 sent:25 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 25) v7 from osd.5 2019-12-07T19:34:02.650+0000 7fe2e753b700 15 osd.0 25 enqueue_peering_evt 1.2s0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(3) notify: (query:25 sent:25 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.650+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(3) notify: (query:25 sent:25 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.650+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 25 ==== pg_notify2(1.2s0 1.2s0 (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e25/25) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e8122b800 con 0x558e818d6800 2019-12-07T19:34:02.650+0000 7fe2e753b700 15 osd.0 25 enqueue_peering_evt 1.2s0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(4) notify: (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.650+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(4) notify: (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.650+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 14 ==== osd_map(25..25 src has 1..25) v4 ==== 386+0+0 (unknown 0 0 0) 0x558e83dd0900 con 0x558e818fe000 2019-12-07T19:34:02.650+0000 7fe2d4a80700 20 osd.0 25 OSD::ms_dispatch: osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.650+0000 7fe2d4a80700 10 osd.0 25 do_waiters -- start 2019-12-07T19:34:02.650+0000 7fe2d4a80700 10 osd.0 25 do_waiters -- finish 2019-12-07T19:34:02.650+0000 7fe2d4a80700 20 osd.0 25 _dispatch 0x558e83dd0900 osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.650+0000 7fe2d4a80700 3 osd.0 25 handle_osd_map epochs [25,25], i have 25, src has [1,25] 2019-12-07T19:34:02.650+0000 7fe2d4a80700 10 osd.0 25 no new maps here, dropping 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 7(4) notify: (query:25 sent:25 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e25) pg 0x558e81075000 2019-12-07T19:34:02.650+0000 7fe2c6a64700 10 osd.0 25 dequeue_op 0x558e819438c0 prio 63 cost 4 latency 0.996645 osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] 2019-12-07T19:34:02.650+0000 7fe2c6a64700 10 osd.0 25 dequeue_op 0x558e819438c0 finish 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(3) notify: (query:25 sent:25 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(3) notify: (query:25 sent:25 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 7(4) notify: (query:25 sent:25 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558e81075000 2019-12-07T19:34:02.650+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 7(4) notify: (query:25 sent:25 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.650+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.7(4) 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:02.650+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:03.256984+0000, must = 0 2019-12-07T19:34:02.650+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 7 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:02.650+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:02.650+0000 7fe2c6a64700 10 osd.0 25 queue_want_up_thru want 25 <= queued 25, currently 24 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(4) notify: (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.650+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(4) notify: (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(3) notify: (query:25 sent:25 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558e81075000 2019-12-07T19:34:02.650+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(3) notify: (query:25 sent:25 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.650+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.5(3) 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:02.650+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:10.988324+0000, must = 0 2019-12-07T19:34:02.650+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:02.650+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:02.650+0000 7fe2c6a64700 10 osd.0 25 queue_want_up_thru want 25 <= queued 25, currently 24 2019-12-07T19:34:02.650+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:02.650+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.650+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(4) notify: (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558e81075000 2019-12-07T19:34:02.650+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 5(4) notify: (query:25 sent:25 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.650+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got dup osd.5(4) info 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25), identical to ours 2019-12-07T19:34:02.650+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:02.650+0000 7fe2caa6c700 10 osd.0 25 queue_want_up_thru want 25 <= queued 25, currently 24 2019-12-07T19:34:02.650+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:02.650+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 59 ==== pg_query2(1.3s5 1.3s5 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e83b09680 con 0x558e818fe400 2019-12-07T19:34:02.650+0000 7fe2e6d3a700 15 osd.0 25 enqueue_peering_evt 1.3s5 epoch_sent: 25 epoch_requested: 25 MQuery 1.3s5 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.650+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.3s5 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.648+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x404ab 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x4ab 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x896 and padding block with 0x76a 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x71896 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 4 cleaned 0 in 0.003101423s (0.000000084s flush + 0.003101339s kv commit) 2019-12-07T19:34:02.651+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 60 ==== pg_notify2(1.2s0 1.2s0 (query:25 sent:25 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e25/25) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e80ff3100 con 0x558e818fe400 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 6 submitting 6 deferred done 0 stable 0 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 2 2019-12-07T19:34:02.651+0000 7fe2e6d3a700 15 osd.0 25 enqueue_peering_evt 1.2s0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 4(1) notify: (query:25 sent:25 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.651+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 4(1) notify: (query:25 sent:25 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2292 < min_flush_size 524288 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3581 < min_flush_size 524288 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4604 < min_flush_size 524288 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5627 < min_flush_size 524288 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 6896 < min_flush_size 524288 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 6919 < min_flush_size 524288 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 6919 < min_flush_size 524288 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x71896~1b07 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x71896 0x71896~1b07 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x41896 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x896 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x39d and padding block with 0xc63 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56300,0x558e81a0b500,0x558e8106c000,0x558e83d4d200> 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 kv_submitted 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56300 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56300 finishing 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56300 onodes 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56300 done 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56300 [] 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 kv_submitted 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b500 2019-12-07T19:34:02.651+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 21 ==== osd_map(24..25 src has 1..25) v4 ==== 1314+0+0 (crc 0 0 0) 0x558e80238fc0 con 0x558e80f39000 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 finishing 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b500 onodes 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b500 done 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:02.651+0000 7fe2d959f700 20 osd.0 25 OSD::ms_dispatch: osd_map(24..25 src has 1..25) v4 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b500 [] 2019-12-07T19:34:02.651+0000 7fe2d959f700 10 osd.0 25 do_waiters -- start 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.651+0000 7fe2d959f700 10 osd.0 25 do_waiters -- finish 2019-12-07T19:34:02.651+0000 7fe2d959f700 20 osd.0 25 _dispatch 0x558e80238fc0 osd_map(24..25 src has 1..25) v4 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 kv_submitted 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c000 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 finishing 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c000 onodes 2019-12-07T19:34:02.651+0000 7fe2d959f700 3 osd.0 25 handle_osd_map epochs [24,25], i have 25, src has [1,25] 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c000 done 2019-12-07T19:34:02.651+0000 7fe2d959f700 10 osd.0 25 no new maps here, dropping 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 kv_submitted 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c000 [] 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d200 kv_submitted 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4d200 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d200 finishing 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4d200 onodes 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4d200 done 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4d500 kv_submitted 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4d200 [] 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:02.651+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.651+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.3s5 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.651+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.651+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.3s5 from 4(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0 2019-12-07T19:34:02.651+0000 7fe2c6263700 10 osd.0 25 handle_pg_query_nopg 1.3s5 2019-12-07T19:34:02.651+0000 7fe2c6263700 10 osd.0 25 pg 1.3s5 dne 2019-12-07T19:34:02.651+0000 7fe2c6263700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify((query:25 sent:25 1.3s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 25) v7 -- 0x558e8403f4a0 con 0x558e818fe400 2019-12-07T19:34:02.651+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:02.651+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.651+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.651+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.651+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.es4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.651+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.651+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.es4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0x558e81a98000 2019-12-07T19:34:02.651+0000 7fe2c8a68700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970133255s) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MQuery 1.es4 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.651+0000 7fe2c8a68700 20 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY pruub 36.072387811s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 2019-12-07T19:34:02.651+0000 7fe2c8a68700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:02.651+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:02.651+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.es0 1.es0 (query:25 sent:25 1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 4->0 ([18,24] all_participants=0(4),1(0),3(1),4(2),5(3),6(5) intervals=([18,23] acting 0(4),1(0),3(1),4(2),5(3),6(5)))) e25/25) v1 -- 0x558e83c9e300 con 0x558e818d7c00 2019-12-07T19:34:02.651+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:02.651+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e8417b200 seq 16 2019-12-07T19:34:02.651+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:02.651+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:02.651+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:02.651+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417b200 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:02.651+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417b200 onodes shared_blobs 2019-12-07T19:34:02.651+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417b200 allocated 0x[] released 0x[] 2019-12-07T19:34:02.651+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 prepare 2019-12-07T19:34:02.651+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417b200 2019-12-07T19:34:02.651+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 io_done 2019-12-07T19:34:02.651+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:02.651+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.651+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 4(1) notify: (query:25 sent:25 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.651+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.651+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 4(1) notify: (query:25 sent:25 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558e81075000 2019-12-07T19:34:02.648+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=25 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:02.651+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 4(1) notify: (query:25 sent:25 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.651+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got dup osd.4(1) info 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25), identical to ours 2019-12-07T19:34:02.651+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:02.651+0000 7fe2caa6c700 10 osd.0 25 queue_want_up_thru want 25 <= queued 25, currently 24 2019-12-07T19:34:02.651+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:02.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=25 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x7339d 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:02.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=25 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 6 cleaned 0 in 0.000435513s (0.000000087s flush + 0.000435426s kv commit) 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:02.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=25 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] new interval newup [3,0,2,4,5,1] newacting [3,0,2,4,5,1] 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:02.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=25 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x7339d~416 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x7339d 0x7339d~416 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2147483647,4,5,1]p3(0) r=1 lpr=25 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] set_last_peering_reset 25 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4339d 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x39d 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x7b3 and padding block with 0x84d 2019-12-07T19:34:02.651+0000 7fe2cb26d700 15 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,24)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4cc00,0x558e8417a000,0x558e81a0bb00,0x558e8195e300,0x558e83d4d500,0x558e83d4c900> 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 kv_submitted 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4cc00 2019-12-07T19:34:02.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(24-24 up [3,0,2147483647,4,5,1](3) acting [3,0,2147483647,4,5,1](3)) up_thru 18 up_from 13 last_epoch_clean 19 interval(24-24 up [3,0,2147483647,4,5,1](3) acting [3,0,2147483647,4,5,1](3)) : primary up 13-18 does not include interval 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 finishing 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4cc00 onodes 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4cc00 done 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a000 kv_submitted 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4cc00 [] 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a000 kv_submitted 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417a000 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a000 finishing 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417a000 onodes 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a000 done 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417a000 [] 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=24 pruub=10.964683064s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] noting past ([18,24] all_participants=0(1),1(5),3(0),4(3),5(4),6(2) intervals=([18,23] acting 0(1),1(5),3(0),4(3),5(4),6(2))) 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 kv_submitted 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0bb00 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 finishing 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0bb00 onodes 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0bb00 done 2019-12-07T19:34:02.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.964683064s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0bb00 [] 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 kv_submitted 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e300 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 finishing 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e300 onodes 2019-12-07T19:34:02.651+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.964683064s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 done 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e300 [] 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 kv_submitted 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4d500 2019-12-07T19:34:02.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=10.964683064s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d1570] 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 finishing 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4d500 onodes 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4d500 done 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4d500 [] 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 kv_submitted 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c900 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 finishing 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c900 onodes 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c900 done 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c900 [] 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:02.651+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.651+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.651+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.651+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x737b3 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:02.651+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000149012s (0.000000082s flush + 0.000148930s kv commit) 2019-12-07T19:34:02.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417b200> 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 kv_submitted 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417b200 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 finishing 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417b200 onodes 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b200 done 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417b200 [] 2019-12-07T19:34:02.651+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:02.651+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:02.652+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 56 ==== pg_notify2(1.2s0 1.2s0 (query:25 sent:25 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966904335s) 5->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e25/25) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e83c9e300 con 0x558e818d7000 2019-12-07T19:34:02.652+0000 7fe2e7d3c700 15 osd.0 25 enqueue_peering_evt 1.2s0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 3(5) notify: (query:25 sent:25 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966904335s) 5->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.652+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 3(5) notify: (query:25 sent:25 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966904335s) 5->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.652+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 57 ==== osd_map(25..25 src has 1..25) v4 ==== 386+0+0 (unknown 0 0 0) 0x558e83f2a6c0 con 0x558e818d7000 2019-12-07T19:34:02.652+0000 7fe2d4a80700 20 osd.0 25 OSD::ms_dispatch: osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.652+0000 7fe2d4a80700 10 osd.0 25 do_waiters -- start 2019-12-07T19:34:02.652+0000 7fe2d4a80700 10 osd.0 25 do_waiters -- finish 2019-12-07T19:34:02.652+0000 7fe2d4a80700 20 osd.0 25 _dispatch 0x558e83f2a6c0 osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.652+0000 7fe2d4a80700 3 osd.0 25 handle_osd_map epochs [25,25], i have 25, src has [1,25] 2019-12-07T19:34:02.652+0000 7fe2d4a80700 10 osd.0 25 no new maps here, dropping 2019-12-07T19:34:02.651+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 36.071368443s (mnow 26.107186438s + 9.964182005s) 2019-12-07T19:34:02.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:02.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:02.652+0000 7fe2cb26d700 1 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] start_peering_interval up [3,0,2147483647,4,5,1] -> [3,0,2,4,5,1], acting [3,0,2147483647,4,5,1] -> [3,0,2,4,5,1], acting_primary 3(0) -> 3, up_primary 3(0) -> 3, role 1 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:02.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:02.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] release_backoffs [1:80000000::::0,1:90000000::::head) 2019-12-07T19:34:02.652+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:02.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:02.652+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:02.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:02.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:02.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:02.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:02.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:02.652+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:02.652+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:02.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:02.652+0000 7fe2cb26d700 20 osd.0 25 advance_pg new pool opts old pool opts 2019-12-07T19:34:02.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] handle_activate_map: 25 2019-12-07T19:34:02.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.964182005s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:02.652+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.962999825s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:02.653684+0000 1 0.004651 2019-12-07T19:34:02.653+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.962999825s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:02.653+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.962999825s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:02.653+0000 7fe2cb26d700 1 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.962999825s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:02.653+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.962999825s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:02.653722+0000 0 0.000000 2019-12-07T19:34:02.653+0000 7fe2cb26d700 5 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.962999825s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:02.653+0000 7fe2cb26d700 20 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.962999825s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 24 while current is 25 2019-12-07T19:34:02.653+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:02.653+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.962999825s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 NullEvt 2019-12-07T19:34:02.653+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:02.653+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.1s0 1.1s0 (query:25 sent:25 1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.962999825s) 1->0 ([18,24] all_participants=0(1),1(5),3(0),4(3),5(4),6(2) intervals=([18,23] acting 0(1),1(5),3(0),4(3),5(4),6(2)))) e25/25) v1 -- 0x558e83c9ce00 con 0x558e818d7000 2019-12-07T19:34:02.653+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:02.653+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e83d4c600 seq 13 2019-12-07T19:34:02.653+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:02.653+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:02.653+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:02.653+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:02.653+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:02.653+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c600 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:02.653+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c600 onodes shared_blobs 2019-12-07T19:34:02.653+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c600 allocated 0x[] released 0x[] 2019-12-07T19:34:02.653+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c600 prepare 2019-12-07T19:34:02.653+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c600 2019-12-07T19:34:02.653+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c600 io_done 2019-12-07T19:34:02.653+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:02.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:02.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:02.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:02.653+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3860332544 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:02.653+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998661) bluestore 86 GiB free (0.959971), bluefs_ratio 0.0399501 2019-12-07T19:34:02.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:02.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:02.653+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.653+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x737b3~50c to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.653+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x737b3 0x737b3~50c to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.653+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.653+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x437b3 2019-12-07T19:34:02.653+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x7b3 2019-12-07T19:34:02.653+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:02.653+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xcbf and padding block with 0x341 2019-12-07T19:34:02.653+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x73cbf 2019-12-07T19:34:02.653+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:02.653+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:02.653+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:02.653+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:02.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000178387s (0.000000083s flush + 0.000178304s kv commit) 2019-12-07T19:34:02.653+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:02.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:02.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4c600> 2019-12-07T19:34:02.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:02.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c600 kv_submitted 2019-12-07T19:34:02.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c600 2019-12-07T19:34:02.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c600 finishing 2019-12-07T19:34:02.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c600 onodes 2019-12-07T19:34:02.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c600 done 2019-12-07T19:34:02.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:02.653+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c600 [] 2019-12-07T19:34:02.653+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.653+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:02.653+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.653+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 3(5) notify: (query:25 sent:25 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966904335s) 5->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 3(5) notify: (query:25 sent:25 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966904335s) 5->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558e81075000 2019-12-07T19:34:02.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 3(5) notify: (query:25 sent:25 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966904335s) 5->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.3(5) 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966904335s) 2019-12-07T19:34:02.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:03.496298+0000, must = 0 2019-12-07T19:34:02.653+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:02.653+0000 7fe2caa6c700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 3 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:02.653+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:02.653+0000 7fe2caa6c700 10 osd.0 25 queue_want_up_thru want 25 <= queued 25, currently 24 2019-12-07T19:34:02.653+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:02.653+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 58 ==== pg_notify2(1.2s0 1.2s0 (query:25 sent:25 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 5->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e25/25) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e83c9c700 con 0x558e818d7000 2019-12-07T19:34:02.653+0000 7fe2e7d3c700 15 osd.0 25 enqueue_peering_evt 1.2s0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 3(5) notify: (query:25 sent:25 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 5->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.653+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 3(5) notify: (query:25 sent:25 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 5->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.653+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 106 ==== pg_query2(1.as3 1.as3 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417c000 con 0x558e818d7c00 2019-12-07T19:34:02.654+0000 7fe2e7d3c700 15 osd.0 25 enqueue_peering_evt 1.as3 epoch_sent: 25 epoch_requested: 25 MQuery 1.as3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.654+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.as3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 3(5) notify: (query:25 sent:25 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 5->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 3(5) notify: (query:25 sent:25 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 5->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558e81075000 2019-12-07T19:34:02.654+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 3(5) notify: (query:25 sent:25 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 5->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.654+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got dup osd.3(5) info 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25), identical to ours 2019-12-07T19:34:02.654+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:02.654+0000 7fe2caa6c700 10 osd.0 25 queue_want_up_thru want 25 <= queued 25, currently 24 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.as3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.as3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0x558e8193b000 2019-12-07T19:34:02.654+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.970584258s) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MQuery 1.as3 from 1(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY pruub 36.073166239s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 2019-12-07T19:34:02.654+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:02.654+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:02.654+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.as0 1.as0 (query:25 sent:25 1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 3->0 ([18,24] all_participants=0(3),1(0),2(2),4(5),5(4),6(1) intervals=([18,23] acting 0(3),1(0),2(2),4(5),5(4),6(1)))) e25/25) v1 -- 0x558e8404ca00 con 0x558e818d7c00 2019-12-07T19:34:02.654+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e8195e900 seq 12 2019-12-07T19:34:02.654+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:02.654+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:02.654+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e900 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e900 onodes shared_blobs 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e900 allocated 0x[] released 0x[] 2019-12-07T19:34:02.654+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 prepare 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e900 2019-12-07T19:34:02.654+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 io_done 2019-12-07T19:34:02.654+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:02.654+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:02.654+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:02.654+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:02.654+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:02.654+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:02.654+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:02.654+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.654+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x73cbf~416 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.654+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x73cbf 0x73cbf~416 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.654+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.654+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x43cbf 2019-12-07T19:34:02.654+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xcbf 2019-12-07T19:34:02.654+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:02.654+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd5 and padding block with 0xf2b 2019-12-07T19:34:02.654+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 59 ==== pg_query2(1.fs1 1.fs1 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417c000 con 0x558e818d7000 2019-12-07T19:34:02.654+0000 7fe2e7d3c700 15 osd.0 25 enqueue_peering_evt 1.fs1 epoch_sent: 25 epoch_requested: 25 MQuery 1.fs1 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.654+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.fs1 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.654+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.654+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.fs1 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.654+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.654+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.fs1 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0x558e81a9b000 2019-12-07T19:34:02.654+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966054638s) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MQuery 1.fs1 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.654+0000 7fe2c425f700 20 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY pruub 36.071369293s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 2019-12-07T19:34:02.654+0000 7fe2c425f700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:02.654+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:02.654+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.fs0 1.fs0 (query:25 sent:25 1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=0(1),1(4),3(0),4(3),5(2),6(5) intervals=([18,23] acting 0(1),1(4),3(0),4(3),5(2),6(5)))) e25/25) v1 -- 0x558e81229c00 con 0x558e818d7000 2019-12-07T19:34:02.654+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:02.654+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e83d4cf00 seq 20 2019-12-07T19:34:02.654+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:02.654+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:02.654+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:02.654+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4cf00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:02.654+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4cf00 onodes shared_blobs 2019-12-07T19:34:02.654+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4cf00 allocated 0x[] released 0x[] 2019-12-07T19:34:02.654+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 prepare 2019-12-07T19:34:02.654+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4cf00 2019-12-07T19:34:02.654+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 io_done 2019-12-07T19:34:02.654+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x740d5 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000843228s (0.000000088s flush + 0.000843140s kv commit) 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x740d5~416 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x740d5 0x740d5~416 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x440d5 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd5 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x4eb and padding block with 0xb15 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e900> 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:02.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 kv_submitted 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e900 2019-12-07T19:34:02.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 finishing 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e900 onodes 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e900 done 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:02.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e900 [] 2019-12-07T19:34:02.655+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:02.655+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:02.655+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 60 ==== pg_query2(1.1s1 1.1s1 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417c000 con 0x558e818d7000 2019-12-07T19:34:02.655+0000 7fe2e7d3c700 15 osd.0 25 enqueue_peering_evt 1.1s1 epoch_sent: 25 epoch_requested: 25 MQuery 1.1s1 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.655+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.1s1 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.655+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.655+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.1s1 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.655+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.655+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.1s1 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0x558e819e4000 2019-12-07T19:34:02.655+0000 7fe2c7265700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.962999825s) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MQuery 1.1s1 from 3(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.655+0000 7fe2c7265700 20 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY pruub 36.071368443s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 2019-12-07T19:34:02.655+0000 7fe2c7265700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:02.655+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:02.655+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.1s0 1.1s0 (query:25 sent:25 1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 1->0 ([18,24] all_participants=0(1),1(5),3(0),4(3),5(4),6(2) intervals=([18,23] acting 0(1),1(5),3(0),4(3),5(4),6(2)))) e25/25) v1 -- 0x558e8404a700 con 0x558e818d7000 2019-12-07T19:34:02.655+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:02.655+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e8417a300 seq 14 2019-12-07T19:34:02.655+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:02.655+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:02.655+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:02.655+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417a300 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:02.655+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417a300 onodes shared_blobs 2019-12-07T19:34:02.655+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417a300 allocated 0x[] released 0x[] 2019-12-07T19:34:02.655+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 prepare 2019-12-07T19:34:02.655+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417a300 2019-12-07T19:34:02.655+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 io_done 2019-12-07T19:34:02.655+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x744eb 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000549334s (0.000000080s flush + 0.000549254s kv commit) 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4cf00> 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:02.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 kv_submitted 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4cf00 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:02.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 finishing 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4cf00 onodes 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4cf00 done 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:02.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4cf00 [] 2019-12-07T19:34:02.655+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.655+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x744eb~416 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x744eb 0x744eb~416 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x444eb 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x4eb 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x901 and padding block with 0x6ff 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x74901 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:02.655+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:02.655+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000233151s (0.000000078s flush + 0.000233073s kv commit) 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:02.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417a300> 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:02.655+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 107 ==== pg_notify2(1.2s0 1.2s0 (query:25 sent:25 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963050445s) 2->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e25/25) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e83c9c700 con 0x558e818d7c00 2019-12-07T19:34:02.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 kv_submitted 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417a300 2019-12-07T19:34:02.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 finishing 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417a300 onodes 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a300 done 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:02.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417a300 [] 2019-12-07T19:34:02.655+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:02.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:02.655+0000 7fe2e7d3c700 15 osd.0 25 enqueue_peering_evt 1.2s0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 1(2) notify: (query:25 sent:25 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963050445s) 2->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.655+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 1(2) notify: (query:25 sent:25 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963050445s) 2->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.655+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 108 ==== pg_notify2(1.2s0 1.2s0 (query:25 sent:25 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e25/25) v1 ==== 1215+0+0 (unknown 0 0 0) 0x558e8404ca00 con 0x558e818d7c00 2019-12-07T19:34:02.655+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.655+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.655+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 1(2) notify: (query:25 sent:25 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963050445s) 2->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.655+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.655+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 1(2) notify: (query:25 sent:25 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963050445s) 2->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558e81075000 2019-12-07T19:34:02.655+0000 7fe2e7d3c700 15 osd.0 25 enqueue_peering_evt 1.2s0 epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 1(2) notify: (query:25 sent:25 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.655+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 1(2) notify: (query:25 sent:25 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) 2019-12-07T19:34:02.655+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 1(2) notify: (query:25 sent:25 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963050445s) 2->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.655+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.1(2) 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963050445s) 2019-12-07T19:34:02.655+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:34:58.202712+0000, must = 0 2019-12-07T19:34:02.655+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:02.655+0000 7fe2caa6c700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 1 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:02.655+0000 7fe2caa6c700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Common peer features: 3f01cfb8ffadffff 2019-12-07T19:34:02.655+0000 7fe2caa6c700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Common acting features: 3f01cfb8ffadffff 2019-12-07T19:34:02.655+0000 7fe2caa6c700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Common upacting features: 3f01cfb8ffadffff 2019-12-07T19:34:02.655+0000 7fe2caa6c700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] exit Started/Primary/Peering/GetInfo 2019-12-07T19:34:02.656665+0000 10 0.000427 2019-12-07T19:34:02.655+0000 7fe2caa6c700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] enter Started/Primary/Peering/GetLog 2019-12-07T19:34:02.655+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:02.656+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 21 ==== pg_query2(1.5s4 1.5s4 query(info 0'0 epoch_sent 25) e25/25) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417ca00 con 0x558e818d6000 2019-12-07T19:34:02.656+0000 7fe2e753b700 15 osd.0 25 enqueue_peering_evt 1.5s4 epoch_sent: 25 epoch_requested: 25 MQuery 1.5s4 from 2(0) query_epoch 25 query: query(info 0'0 epoch_sent 25) 2019-12-07T19:34:02.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.0(0) 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2019-12-07T19:34:02.656+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.5s4 from 2(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) 2019-12-07T19:34:02.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.1(2) 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.963050445s) 2019-12-07T19:34:02.656+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.3(5) 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.966904335s) 2019-12-07T19:34:02.656+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.5s4 from 2(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.656+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:02.656+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MQuery 1.5s4 from 2(0) query_epoch 25 query: query(info 0'0 epoch_sent 25)) prio 255 cost 10 e25) pg 0 2019-12-07T19:34:02.656+0000 7fe2c9269700 10 osd.0 25 handle_pg_query_nopg 1.5s4 2019-12-07T19:34:02.656+0000 7fe2c9269700 10 osd.0 25 pg 1.5s4 dne 2019-12-07T19:34:02.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.4(1) 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.971150263s) 2019-12-07T19:34:02.656+0000 7fe2c9269700 10 osd.0 25 send_incremental_map 24 -> 25 to 0x558e818d6000 v2:172.21.15.131:6804/37077 2019-12-07T19:34:02.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.5(3) 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:02.656+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(25..25 src has 1..25) v4 -- 0x558e817df680 con 0x558e818d6000 2019-12-07T19:34:02.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.5(4) 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25 pruub=9.972591842s) 2019-12-07T19:34:02.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.7(4) 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:02.656+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:02.656+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 1(2) notify: (query:25 sent:25 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) queued 2019-12-07T19:34:02.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] For position 0: selecting up[i]: 0(0) For position 1: selecting up[i]: 4(1) For position 2: selecting up[i]: 1(2) For position 3: selecting up[i]: 5(3) For position 4: selecting up[i]: 7(4) For position 5: selecting up[i]: 3(5) 2019-12-07T19:34:02.656+0000 7fe2c9269700 10 osd.0 25 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 24 -> 25 (shared) 2019-12-07T19:34:02.656+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify((query:25 sent:25 1.5s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 25) v7 -- 0x558e819c5860 con 0x558e818d6000 2019-12-07T19:34:02.656+0000 7fe2caa6c700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_async_recovery_ec candidates by cost are: 10,5(3),10,7(4) 2019-12-07T19:34:02.656+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:02.656+0000 7fe2caa6c700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_async_recovery_ec result want=[0,4,1,2147483647,2147483647,3] async_recovery=5(3),7(4) 2019-12-07T19:34:02.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting want [0,4,1,2147483647,2147483647,3] != acting [0,4,1,5,7,3], requesting pg_temp change 2019-12-07T19:34:02.656+0000 7fe2caa6c700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] exit Started/Primary/Peering/GetLog 2019-12-07T19:34:02.656876+0000 0 0.000000 2019-12-07T19:34:02.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] state: Leaving Peering 2019-12-07T19:34:02.656+0000 7fe2caa6c700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 peering mbc={} ps=[2~1]] exit Started/Primary/Peering 2019-12-07T19:34:02.656897+0000 0 0.000000 2019-12-07T19:34:02.656+0000 7fe2caa6c700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] exit Started/Primary 2019-12-07T19:34:02.656907+0000 0 0.000000 2019-12-07T19:34:02.656+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 22 ==== osd_map(25..25 src has 1..25) v4 ==== 386+0+0 (unknown 0 0 0) 0x558e83dd06c0 con 0x558e818d6000 2019-12-07T19:34:02.656+0000 7fe2d4a80700 20 osd.0 25 OSD::ms_dispatch: osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.656+0000 7fe2d4a80700 10 osd.0 25 do_waiters -- start 2019-12-07T19:34:02.656+0000 7fe2d4a80700 10 osd.0 25 do_waiters -- finish 2019-12-07T19:34:02.656+0000 7fe2d4a80700 20 osd.0 25 _dispatch 0x558e83dd06c0 osd_map(25..25 src has 1..25) v4 2019-12-07T19:34:02.656+0000 7fe2caa6c700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:02.656+0000 7fe2d4a80700 3 osd.0 25 handle_osd_map epochs [25,25], i have 25, src has [1,25] 2019-12-07T19:34:02.656+0000 7fe2d4a80700 10 osd.0 25 no new maps here, dropping 2019-12-07T19:34:02.656+0000 7fe2caa6c700 20 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:02.656+0000 7fe2caa6c700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] enter Started/Primary 2019-12-07T19:34:02.656+0000 7fe2caa6c700 5 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] enter Started/Primary/WaitActingChange 2019-12-07T19:34:02.656+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:02.656+0000 7fe2caa6c700 10 osd.0 25 queue_want_up_thru want 25 <= queued 25, currently 24 2019-12-07T19:34:02.656+0000 7fe2caa6c700 10 osd.0 25 send_pg_temp {1.2=[0,4,1,2147483647,2147483647,3]} 2019-12-07T19:34:02.656+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- osd_pgtemp(e25 {1.2=[0,4,1,2147483647,2147483647,3]} v25) v2 -- 0x558e83ff1b00 con 0x558e80f39000 2019-12-07T19:34:02.656+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:02.656+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:02.656+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25)> waiting <> waiting_peering {} 2019-12-07T19:34:02.656+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 1(2) notify: (query:25 sent:25 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e25) pg 0x558e81075000 2019-12-07T19:34:02.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] do_peering_event: epoch_sent: 25 epoch_requested: 25 MNotifyRec 1.2s0 from 1(2) notify: (query:25 sent:25 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) 2->0 ([18,24] all_participants=0(0),1(2),3(5),4(1),5(4),6(3) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:02.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] state: In WaitActingChange, ignoring MNotifyRec 2019-12-07T19:34:02.656+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:02.656+0000 7fe2c6a64700 10 osd.0 25 queue_want_up_thru want 25 <= queued 25, currently 24 2019-12-07T19:34:02.656+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:02.656+0000 7fe2e2ca2700 10 osd.0 25 tick_without_osd_lock 2019-12-07T19:34:02.656+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fd5a000/0x40000000/0x1680000000, data 0x27a7ec/0x2a4000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:02.656+0000 7fe2e2ca2700 25 osd.0 25 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254252+0000 last_rx_front 2019-12-07T19:34:02.254310+0000 2019-12-07T19:34:02.656+0000 7fe2e2ca2700 25 osd.0 25 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254273+0000 last_rx_front 2019-12-07T19:34:02.254232+0000 2019-12-07T19:34:02.656+0000 7fe2e2ca2700 25 osd.0 25 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254209+0000 last_rx_front 2019-12-07T19:34:02.254194+0000 2019-12-07T19:34:02.656+0000 7fe2e2ca2700 25 osd.0 25 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254409+0000 last_rx_front 2019-12-07T19:34:02.254290+0000 2019-12-07T19:34:02.656+0000 7fe2e2ca2700 25 osd.0 25 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254354+0000 last_rx_front 2019-12-07T19:34:02.254301+0000 2019-12-07T19:34:02.656+0000 7fe2e2ca2700 25 osd.0 25 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254368+0000 last_rx_front 2019-12-07T19:34:02.254333+0000 2019-12-07T19:34:02.656+0000 7fe2e2ca2700 20 osd.0 25 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:02.656+0000 7fe2e2ca2700 10 osd.0 25 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:02.656+0000 7fe2e2ca2700 20 osd.0 25 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:02.656+0000 7fe2e2ca2700 10 osd.0 25 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:02.700+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.720+0000 7fe2e4535700 10 osd.0 25 tick 2019-12-07T19:34:02.720+0000 7fe2e4535700 10 osd.0 25 maybe_update_heartbeat_peers updating 2019-12-07T19:34:02.720+0000 7fe2e4535700 10 osd.0 25 adding neighbor peer osd.1 2019-12-07T19:34:02.720+0000 7fe2e4535700 10 osd.0 25 adding neighbor peer osd.5 2019-12-07T19:34:02.720+0000 7fe2e4535700 10 osd.0 25 adding neighbor peer osd.7 2019-12-07T19:34:02.720+0000 7fe2e4535700 10 osd.0 25 adding random peer osd.3 2019-12-07T19:34:02.720+0000 7fe2e4535700 10 osd.0 25 adding random peer osd.4 2019-12-07T19:34:02.720+0000 7fe2e4535700 10 osd.0 25 maybe_update_heartbeat_peers 6 peers, extras 1,2,3,4,5,7 2019-12-07T19:34:02.720+0000 7fe2e4535700 10 osd.0 25 do_waiters -- start 2019-12-07T19:34:02.720+0000 7fe2e4535700 10 osd.0 25 do_waiters -- finish 2019-12-07T19:34:02.720+0000 7fe2e4535700 20 osd.0 25 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:02.750+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.800+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.850+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.859+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83d68800 0x558e80ef9080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:02.859+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83d68800 0x558e80ef9080 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:02.859+0000 7fe2e753b700 10 osd.0 25 ms_handle_authentication new session 0x558e818e96c0 con 0x558e83d68800 entity client.admin addr 2019-12-07T19:34:02.859+0000 7fe2e753b700 10 osd.0 25 ms_handle_authentication session 0x558e818e96c0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:02.859+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1006309048 conn(0x558e83d68800 0x558e80ef9080 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=client.4591 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:02.859+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4591 172.21.15.131:0/1006309048 1 ==== command(tid 1: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558e819c5860 con 0x558e83d68800 2019-12-07T19:34:02.859+0000 7fe2d959f700 20 osd.0 25 OSD::ms_dispatch: command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:02.859+0000 7fe2d959f700 10 osd.0 25 do_waiters -- start 2019-12-07T19:34:02.859+0000 7fe2d959f700 10 osd.0 25 do_waiters -- finish 2019-12-07T19:34:02.859+0000 7fe2d959f700 20 osd.0 25 _dispatch 0x558e819c5860 command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:02.859+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/1006309048 -- command_reply(tid 1: 0 ) v1 -- 0x558e8022f340 con 0x558e83d68800 2019-12-07T19:34:02.880+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4591 172.21.15.131:0/1006309048 2 ==== command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558e817b43c0 con 0x558e83d68800 2019-12-07T19:34:02.880+0000 7fe2d959f700 20 osd.0 25 OSD::ms_dispatch: command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:02.880+0000 7fe2d959f700 10 osd.0 25 do_waiters -- start 2019-12-07T19:34:02.880+0000 7fe2d959f700 10 osd.0 25 do_waiters -- finish 2019-12-07T19:34:02.880+0000 7fe2d959f700 20 osd.0 25 _dispatch 0x558e817b43c0 command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:02.883+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/1006309048 -- command_reply(tid 2: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558e8022fdc0 con 0x558e83d68800 2019-12-07T19:34:02.900+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:02.950+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.001+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.044+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1006309048 conn(0x558e83d68800 msgr2=0x558e80ef9080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 61 2019-12-07T19:34:03.044+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1006309048 conn(0x558e83d68800 msgr2=0x558e80ef9080 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:03.044+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1006309048 conn(0x558e83d68800 0x558e80ef9080 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:03.044+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1006309048 conn(0x558e83d68800 0x558e80ef9080 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:03.044+0000 7fe2d959f700 2 osd.0 25 ms_handle_reset con 0x558e83d68800 session 0x558e818e96c0 2019-12-07T19:34:03.051+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.101+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.151+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.201+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.251+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.302+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.352+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.372+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.7 172.21.15.109:0/36939 8 ==== osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.808738428s delta_ub 18446744073.690501011s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84028200 con 0x558e81749000 2019-12-07T19:34:03.372+0000 7fe2e6d3a700 20 osd.0 25 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.689932589s,18446744073.690501011s]) 2019-12-07T19:34:03.372+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.828357455s delta_ub 0.019619027s) v5 -- 0x558e83ca8a00 con 0x558e81749000 2019-12-07T19:34:03.373+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.7 172.21.15.109:0/36939 8 ==== osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.808738428s delta_ub 18446744073.690501011s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84028200 con 0x558e81748c00 2019-12-07T19:34:03.373+0000 7fe2e6d3a700 20 osd.0 25 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.689810581s,18446744073.690501011s]) 2019-12-07T19:34:03.373+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.373299+0000/26.808738428s send_stamp 26.828479463s delta_ub 0.019741035s) v5 -- 0x558e83ca8200 con 0x558e81748c00 2019-12-07T19:34:03.402+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.442+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.2 172.21.15.131:0/37077 7 ==== osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.443125+0000/26.914843615s send_stamp 26.914843615s delta_ub 0.017223111s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81239c00 con 0x558e8182c000 2019-12-07T19:34:03.442+0000 7fe2e6d3a700 20 osd.0 25 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016922604s,0.017223111s]) 2019-12-07T19:34:03.442+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.443125+0000/26.914843615s send_stamp 26.897921011s delta_ub 18446744073.692629012s) v5 -- 0x558e83ca8200 con 0x558e8182c000 2019-12-07T19:34:03.442+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.2 172.21.15.131:0/37077 7 ==== osd_ping(ping e25 up_from 13 ping_stamp 2019-12-07T19:34:03.443125+0000/26.914843615s send_stamp 26.914843615s delta_ub 0.017223111s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83ca8600 con 0x558e8182d400 2019-12-07T19:34:03.442+0000 7fe2e7d3c700 20 osd.0 25 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016890965s,0.017223111s]) 2019-12-07T19:34:03.442+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e25 up_from 13 ping_stamp 2019-12-07T19:34:03.443125+0000/26.914843615s send_stamp 26.897952650s delta_ub 18446744073.692660651s) v5 -- 0x558e83ca8400 con 0x558e8182d400 2019-12-07T19:34:03.452+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.502+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.552+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 292784 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.601+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 61 ==== pg_notify2(1.2s0 1.2s0 (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e26/26) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e80ff3100 con 0x558e818fe400 2019-12-07T19:34:03.601+0000 7fe2e6d3a700 15 osd.0 25 enqueue_peering_evt 1.2s0 epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 4(1) notify: (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.601+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 4(1) notify: (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.601+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 4(1) notify: (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.601+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26)> waiting <> waiting_peering {} 2019-12-07T19:34:03.601+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 4(1) notify: (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) pg 0x558e81075000 2019-12-07T19:34:03.601+0000 7fe2caa6c700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 26, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 4(1) notify: (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:03.601+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 62 ==== osd_map(26..26 src has 1..26) v4 ==== 913+0+0 (unknown 0 0 0) 0x558e83dd1200 con 0x558e818fe400 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 osd.0 25 OSD::ms_dispatch: osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.601+0000 7fe2d4a80700 10 osd.0 25 do_waiters -- start 2019-12-07T19:34:03.601+0000 7fe2d4a80700 10 osd.0 25 do_waiters -- finish 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 osd.0 25 _dispatch 0x558e83dd1200 osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.601+0000 7fe2d4a80700 3 osd.0 25 handle_osd_map epochs [26,26], i have 25, src has [1,26] 2019-12-07T19:34:03.601+0000 7fe2d4a80700 10 osd.0 25 handle_osd_map got inc map for epoch 26 2019-12-07T19:34:03.601+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3820e826:::osdmap.25:0# 0x0~0 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1428 size 0x1428 (5160) 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194fdc0 blob([0x105000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194fce0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194fc70 blob([!~1000,0x106000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,428]) SharedBlob(0x558e83ac5f10 sbid 0x0)) need 0x1000~428 cache has 0x[1000~428] 2019-12-07T19:34:03.601+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3820e826:::osdmap.25:0# 0x0~1428 = 5160 2019-12-07T19:34:03.601+0000 7fe2d4a80700 10 osd.0 25 add_map_bl 25 5160 bytes 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 osd.0 25 got_full_map 26, nothing requested 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 osd.0 25 handle_osd_map pg_num_history pg_num_history(e26 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:03.601+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps purged_snaps {26={}} 2019-12-07T19:34:03.601+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:03.601+0000 7fe2d4a80700 10 osd.0 25 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e26 [1,26] lci=[13,26]) 2019-12-07T19:34:03.601+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e819a0300 seq 18 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:cb3b972f:::inc_osdmap.26:0# key 0x7f7fffffffffffffffcb3b97'/!inc_osdmap.26!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:03.601+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 63 ==== pg_query2(1.bs1 1.bs1 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417c280 con 0x558e818fe400 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:03.601+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:cb3b972f:::inc_osdmap.26:0# 0x0~365 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1115 2019-12-07T19:34:03.601+0000 7fe2e6d3a700 15 osd.0 25 enqueue_peering_evt 1.bs1 epoch_sent: 26 epoch_requested: 26 MQuery 1.bs1 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:cb3b972f:::inc_osdmap.26:0# 0x0~365 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:03.601+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs1 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:03.601+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~365 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xc9b on front/back, now 0x0~1000 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0300 1 blobs 2019-12-07T19:34:03.601+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 64 ==== pg_query2(1.bs3 1.bs3 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417c280 con 0x558e818fe400 2019-12-07T19:34:03.601+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:03.601+0000 7fe2e6d3a700 15 osd.0 25 enqueue_peering_evt 1.bs3 epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.601+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.601+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x107000~1000/1000,1000,0 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x107000~1000] 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e840142a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84014540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e840142a0 blob([0x107000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84014540 sbid 0x0)) 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore.blob(0x558e840142a0) get_ref 0x0~365 Blob(0x558e840142a0 blob([0x107000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84014540 sbid 0x0)) 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore.blob(0x558e840142a0) get_ref init 0x1000, 1000 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~365: 0x0~365 Blob(0x558e840142a0 blob([0x107000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x365) SharedBlob(0x558e84014540 sbid 0x0)) 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e84014558 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x365 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:03.601+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83f29c88) dirty_range mark inline shard dirty 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:cb3b972f:::inc_osdmap.26:0# 0x0~365 = 0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:35a0e826:::osdmap.26:0# key 0x7f7fffffffffffffff35a0e8'&!osdmap.26!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:35a0e826:::osdmap.26:0# 0x0~1671 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1116 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:35a0e826:::osdmap.26:0# 0x0~1671 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~671 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0x98f on front/back, now 0x0~1000 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0300 2 blobs 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x108000~2000/1000,2000,0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x108000~2000] 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f942a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f94460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f942a0 blob([0x108000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f94460 sbid 0x0)) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.blob(0x558e83f942a0) get_ref 0x0~1000 Blob(0x558e83f942a0 blob([0x108000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f94460 sbid 0x0)) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.blob(0x558e83f942a0) get_ref init 0x1000, 1000 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e83f942a0 blob([0x108000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f94460 sbid 0x0)) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e83f94478 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f94540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f94cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f94540 blob([!~1000,0x109000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f94cb0 sbid 0x0)) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.blob(0x558e83f94540) get_ref 0x1000~671 Blob(0x558e83f94540 blob([!~1000,0x109000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e83f94cb0 sbid 0x0)) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.blob(0x558e83f94540) get_ref init 0x2000, 1000 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~671: 0x1000~671 Blob(0x558e83f94540 blob([!~1000,0x109000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,671]) SharedBlob(0x558e83f94cb0 sbid 0x0)) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e83f94cc8 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1671 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8402f788) dirty_range mark inline shard dirty 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:35a0e826:::osdmap.26:0# 0x0~1671 = 0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0300 0 blobs 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0300 0 blobs 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0300 cost 24095 (4 ios * 4000 + 8095 bytes) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0300 onodes 0x558e80251b80,0x558e817b9180,0x558e83f29b80,0x558e8402f680 shared_blobs 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83f29c88) update #-1:cb3b972f:::inc_osdmap.26:0# 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83f29c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:cb3b972f:::inc_osdmap.26:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8402f788) update #-1:35a0e826:::osdmap.26:0# 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8402f788) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:35a0e826:::osdmap.26:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0300 allocated 0x[107000~3000] released 0x[] 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 freelist allocate 0x107000~3000 2019-12-07T19:34:03.602+0000 7fe2d4a80700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0300 prepare 2019-12-07T19:34:03.602+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e819a0300 2019-12-07T19:34:03.602+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs1 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.602+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs1 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0 2019-12-07T19:34:03.602+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 map 26 > 25 2019-12-07T19:34:03.602+0000 7fe2c6263700 20 osd.0 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 26, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs1 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.602+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.602+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0x558e81928000 2019-12-07T19:34:03.602+0000 7fe2c6263700 20 osd.0 op_wq _add_slot_waiter 1.bs3 peering, item epoch is 26, will wait on OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.602+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:03.602+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0300 aio_wait 2019-12-07T19:34:03.602+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0300 2019-12-07T19:34:03.602+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0300 io_done 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:03.602+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 65 ==== pg_query2(1.3s5 1.3s5 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417c280 con 0x558e818fe400 2019-12-07T19:34:03.602+0000 7fe2e6d3a700 15 osd.0 25 enqueue_peering_evt 1.3s5 epoch_sent: 26 epoch_requested: 26 MQuery 1.3s5 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.602+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.3s5 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.602+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:03.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.3s5 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83f29b80 had 1 2019-12-07T19:34:03.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402f680 had 1 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:03.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.3s5 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0 2019-12-07T19:34:03.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 map 26 > 25 2019-12-07T19:34:03.602+0000 7fe2ca26b700 20 osd.0 op_wq _add_slot_waiter 1.3s5 peering, item epoch is 26, will wait on OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.3s5 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.602+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:03.602+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:03.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:03.602+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.602+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x74901~226c to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.602+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x74901 0x74901~226c to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x44901 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x901 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb6d and padding block with 0x493 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x76b6d 2019-12-07T19:34:03.602+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:03.602+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:03.602+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000143361s (0.000004882s flush + 0.000138479s kv commit) 2019-12-07T19:34:03.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:03.602+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:03.602+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a0300> 2019-12-07T19:34:03.602+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:03.602+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0300 kv_submitted 2019-12-07T19:34:03.602+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0300 2019-12-07T19:34:03.602+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e819a0300 osr 0x558e8107a500 2019-12-07T19:34:03.602+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) _discard drop 15 0x1b000~1000 2019-12-07T19:34:03.602+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) prepare_write seq 16 0x1b000~1000 crc 9316858d 2019-12-07T19:34:03.602+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) _discard drop 15 0x2000~1000 2019-12-07T19:34:03.602+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) prepare_write seq 16 0x2000~1000 crc 6c6cdd8e 2019-12-07T19:34:03.602+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:03.602+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 1 osrs, 4 txcs 2019-12-07T19:34:03.602+0000 7fe2dc28f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock osr 0x558e8107a500 2 ios pending 2019-12-07T19:34:03.602+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 16 0x2000~1000 2019-12-07T19:34:03.602+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x2000~1000 crc 6c6cdd8e 2019-12-07T19:34:03.602+0000 7fe2d7285700 10 osd.0 25 _committed_osd_maps 26..26 2019-12-07T19:34:03.602+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 16 0x1b000~1000 2019-12-07T19:34:03.602+0000 7fe2d7285700 10 osd.0 25 advance to epoch 26 (<= last 26 <= newest_map 26) 2019-12-07T19:34:03.602+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x1b000~1000 crc 9316858d 2019-12-07T19:34:03.602+0000 7fe2d7285700 7 osd.0 26 consume_map version 26 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:0.identify_splits_and_merges identify_splits_and_merges slot 1.8s5 has no pg and waiting_for_split 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0 26 identify_splits_and_merges 1.9s3 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0 26 identify_splits_and_merges 1.1s1 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0 26 identify_splits_and_merges 1.as3 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0 26 identify_splits_and_merges 1.2s0 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:3.identify_splits_and_merges identify_splits_and_merges slot 1.3s5 has no pg and waiting_for_split 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:3.identify_splits_and_merges identify_splits_and_merges slot 1.bs1 has no pg and waiting_for_split 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0 26 identify_splits_and_merges 1.bs3 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0 26 identify_splits_and_merges 1.4s1 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:5.identify_splits_and_merges identify_splits_and_merges slot 1.5s4 has no pg and waiting_for_split 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0 26 identify_splits_and_merges 1.ds5 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0 26 identify_splits_and_merges 1.es4 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0 26 identify_splits_and_merges 1.7s3 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0 26 identify_splits_and_merges 1.fs1 e25 to e26 pg_nums {18=16} 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0 26 prune_pg_created 2019-12-07T19:34:03.602+0000 7fe2d7285700 10 osd.0:0.consume_map 26 (was 25) 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 empty, pruning 2019-12-07T19:34:03.602+0000 7fe2d7285700 10 osd.0:1.consume_map 26 (was 25) 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:03.602+0000 7fe2d7285700 10 osd.0:2.consume_map 26 (was 25) 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 pending_peering first epoch 26 <= 26, requeueing 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.2s0 to_process <> waiting <> waiting_peering {26=0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26)>} 2019-12-07T19:34:03.602+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 296020 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.602+0000 7fe2d7285700 10 osd.0:3.consume_map 26 (was 25) 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 pending_peering first epoch 26 <= 26, requeueing 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.3s5 to_process <> waiting <> waiting_peering {26=} 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 pending_peering first epoch 26 <= 26, requeueing 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {26=} 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 2019-12-07T19:34:03.602+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish osr 0x558e8107a500 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 pending_peering first epoch 26 <= 26, requeueing 2019-12-07T19:34:03.602+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish dequeueing 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.bs3 to_process <> waiting <> waiting_peering {26=} 2019-12-07T19:34:03.602+0000 7fe2d7285700 10 osd.0:4.consume_map 26 (was 25) 2019-12-07T19:34:03.602+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:03.602+0000 7fe2d7285700 10 osd.0:5.consume_map 26 (was 25) 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 empty, pruning 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:03.603+0000 7fe2d7285700 10 osd.0:6.consume_map 26 (was 25) 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:03.603+0000 7fe2d7285700 10 osd.0:7.consume_map 26 (was 25) 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:03.603+0000 7fe2d7285700 10 osd.0 26 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:03.603+0000 7fe2d7285700 15 osd.0 26 enqueue_peering_evt 1.9s3 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.603+0000 7fe2d7285700 15 osd.0 26 enqueue_peering_evt 1.1s1 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.603+0000 7fe2d7285700 15 osd.0 26 enqueue_peering_evt 1.as3 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.603+0000 7fe2d7285700 15 osd.0 26 enqueue_peering_evt 1.2s0 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.603+0000 7fe2d7285700 15 osd.0 26 enqueue_peering_evt 1.bs3 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.603+0000 7fe2d7285700 15 osd.0 26 enqueue_peering_evt 1.4s1 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.603+0000 7fe2d7285700 15 osd.0 26 enqueue_peering_evt 1.ds5 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.603+0000 7fe2d7285700 15 osd.0 26 enqueue_peering_evt 1.es4 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.603+0000 7fe2d7285700 15 osd.0 26 enqueue_peering_evt 1.7s3 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.603+0000 7fe2d7285700 15 osd.0 26 enqueue_peering_evt 1.fs1 epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.603+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) 2019-12-07T19:34:03.603+0000 7fe2d7285700 7 osd.0 26 activate_map version 26 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558e81938000 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 25 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 20 osd.0:4.update_pg_epoch 1.4s1 25 -> 26 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 25 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 26 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 26 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 20 osd.0 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 26 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 26 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 26 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 26 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 26 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:03.603+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558e8122e000 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.7s3 25 -> 26 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 25 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2] -- 1/1 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [1,4,3,0,7,2] newacting [1,2147483647,2147483647,0,7,2] 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:03.603+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:03.604254+0000 3 0.000077 2019-12-07T19:34:03.603+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:03.604263+0000 0 0.000000 2019-12-07T19:34:03.603+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:03.603+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:03.603+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558e819e7000 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [1,4,3,0,7,2] newacting [1,2147483647,2147483647,0,7,2] 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.603+0000 7fe2c8267700 15 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'9 unknown NOTIFY mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:03.603+0000 7fe2c5261700 10 osd.0 pg_epoch: 25 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.603+0000 7fe2c5261700 20 osd.0:5.update_pg_epoch 1.ds5 25 -> 26 2019-12-07T19:34:03.603+0000 7fe2c5261700 10 osd.0 pg_epoch: 25 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(25-25 up [1,4,3,0,7,2](1) acting [1,4,3,0,7,2](1)) up_thru 24 up_from 13 last_epoch_clean 19 interval(25-25 up [1,4,3,0,7,2](1) acting [1,4,3,0,7,2](1)) : primary up 13-24 does not include interval 2019-12-07T19:34:03.603+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.603+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558e81a98000 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] noting past ([18,25] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4))) 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:03.603+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 4(1) notify: (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:03.603+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26)> waiting <> waiting_peering {} 2019-12-07T19:34:03.603+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 4(1) notify: (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) pg 0x558e81075000 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:03.603+0000 7fe2c8a68700 20 osd.0:6.update_pg_epoch 1.es4 25 -> 26 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 27.059078442s + 0s) 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 25 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map [1,3,4,5,0,2]/[1,3,4,5,0,2147483647] -- 1/1 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:03.603+0000 7fe2c8a68700 20 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [1,3,4,5,0,2] newacting [1,3,4,5,0,2147483647] 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.603+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.2s0 25 -> 26 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:03.603+0000 7fe2c8267700 1 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval up [1,4,3,0,7,2] -> [1,4,3,0,7,2], acting [1,4,3,0,7,2] -> [1,2147483647,2147483647,0,7,2], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 3 -> 3, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3] -- 0/0 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:03.603+0000 7fe2c8a68700 5 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:03.604418+0000 3 0.000094 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] state: verifying no want_acting [] targets didn't go down 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] release_backoffs [1:e0000000::::0,1:f0000000::::head) 2019-12-07T19:34:03.603+0000 7fe2c8a68700 5 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:03.604428+0000 0 0.000000 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:03.603+0000 7fe2c8a68700 5 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:03.603+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] new interval newup [0,4,1,5,7,3] newacting [0,4,1,2147483647,2147483647,3] 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.603+0000 7fe2c8a68700 20 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=25 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:03.603+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] exit Started/Primary/WaitActingChange 2019-12-07T19:34:03.604451+0000 2 0.000358 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:03.603+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] exit Started/Primary 2019-12-07T19:34:03.604462+0000 0 0.000000 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:03.603+0000 7fe2c8a68700 20 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [1,3,4,5,0,2] newacting [1,3,4,5,0,2147483647] 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:03.603+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.603+0000 7fe2c8a68700 20 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.603+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:03.604486+0000 0 0.000000 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:03.603+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:03.603+0000 7fe2c8a68700 15 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,25)/1 crt=21'4 unknown NOTIFY mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:03.603+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=25 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=26 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=26 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=26 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(25-25 up [1,3,4,5,0,2](1) acting [1,3,4,5,0,2](1)) up_thru 24 up_from 13 last_epoch_clean 19 interval(25-25 up [1,3,4,5,0,2](1) acting [1,3,4,5,0,2](1)) : primary up 13-24 does not include interval 2019-12-07T19:34:03.603+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=26 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] new interval newup [0,4,1,5,7,3] newacting [0,4,1,2147483647,2147483647,3] 2019-12-07T19:34:03.603+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:03.604531+0000 1 0.000274 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=26 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] noting past ([18,25] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),6(5) intervals=([18,23] acting 0(4),1(0),3(1),4(2),5(3),6(5))) 2019-12-07T19:34:03.603+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]p0(0) r=0 lpr=26 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 26 clear_ready_to_merge 1.2s0 2019-12-07T19:34:03.603+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:03.603+0000 7fe2c8a68700 20 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:03.603+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:03.603+0000 7fe2c8a68700 20 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:03.603+0000 7fe2c6a64700 15 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,25)/1 crt=23'10 lcod 21'9 mlcod 0'0 unknown mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:03.603+0000 7fe2c8267700 1 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:03.603+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:03.604564+0000 0 0.000000 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 27.059259688s + 0s) 2019-12-07T19:34:03.603+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 25 while current is 26 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(25-25 up [0,4,1,5,7,3](0) acting [0,4,1,5,7,3](0)) up_thru 24 up_from 13 last_epoch_clean 19 interval(25-25 up [0,4,1,5,7,3](0) acting [0,4,1,5,7,3](0)) : primary up 13-24 does not include interval 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] noting past ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) 2019-12-07T19:34:03.603+0000 7fe2c8a68700 1 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval up [1,3,4,5,0,2] -> [1,3,4,5,0,2], acting [1,3,4,5,0,2] -> [1,3,4,5,0,2147483647], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:03.603+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:03.603+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 26 send_incremental_map 25 -> 26 to 0x558e818d7c00 v2:172.21.15.131:6823/37078 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] release_backoffs [1:70000000::::0,1:80000000::::head) 2019-12-07T19:34:03.603+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:cb3b972f:::inc_osdmap.26:0# 0x0~0 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d0af0,0x558e811d01c0,0x558e811d1570] 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~365 size 0x365 (869) 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:03.603+0000 7fe2c8a68700 20 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 27.059313350s + 0s) 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840142a0 blob([0x107000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x365) SharedBlob(0x558e84014540 sbid 0x0)) need 0x0~365 cache has 0x[0~365] 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:cb3b972f:::inc_osdmap.26:0# 0x0~365 = 869 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 26 add_map_inc_bl 26 869 bytes 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:03.603+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(26..26 src has 1..26) v4 -- 0x558e83d46240 con 0x558e818d7c00 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 osd.0 26 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 25 -> 26 (shared) 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:04.201121+0000, must = 0 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:03.603+0000 7fe2c6a64700 1 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] start_peering_interval up [0,4,1,5,7,3] -> [0,4,1,5,7,3], acting [0,4,1,5,7,3] -> [0,4,1,2147483647,2147483647,3], acting_primary 0(0) -> 0, up_primary 0(0) -> 0, role 0 -> 0, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:03.603+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:26 sent:26 1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 3->0 ([18,25] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e26/26) v1 -- 0x558e8404bc00 con 0x558e818d7c00 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:03.603+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.603+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e819a1b00 seq 26 2019-12-07T19:34:03.603+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:03.603+0000 7fe2c8a68700 20 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:03.603+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:03.603+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:03.603+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:03.603+0000 7fe2c8a68700 20 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.604+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1b00 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:03.604+0000 7fe2c8a68700 20 osd.0 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1b00 onodes shared_blobs 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] requeue_op 0x558e819438c0 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1b00 allocated 0x[] released 0x[] 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e26) 2019-12-07T19:34:03.604+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 prepare 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1b00 2019-12-07T19:34:03.604+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 io_done 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] requeue_op 0x558e81943080 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e26) 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] requeue_op 0x558e819431e0 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e26) 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.604+0000 7fe2c8a68700 5 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:03.604729+0000 1 0.000308 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558e81a9b000 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:03.604+0000 7fe2c8a68700 5 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:03.604+0000 7fe2c8a68700 5 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:03.604+0000 7fe2c8a68700 1 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.fs1 25 -> 26 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:03.604+0000 7fe2c8a68700 5 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:03.604762+0000 0 0.000000 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 25 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map [3,0,5,4,1,2]/[3,0,5,4,1,2147483647] -- 3/3 2019-12-07T19:34:03.604+0000 7fe2c6a64700 15 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] publish_stats_to_osd 25:37 2019-12-07T19:34:03.604+0000 7fe2c8a68700 5 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:03.604+0000 7fe2c8a68700 20 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 25 while current is 26 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [3,0,5,4,1,2] newacting [3,0,5,4,1,2147483647] 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:03.604+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:03.604795+0000 3 0.000072 2019-12-07T19:34:03.604+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] [0,4,1,5,7,3] -> [0,4,1,2147483647,2147483647,3], replicas changed 2019-12-07T19:34:03.604+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:03.604804+0000 0 0.000000 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.604+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.604+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.604+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.604+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.es0 1.es0 (query:26 sent:26 1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),6(5) intervals=([18,23] acting 0(4),1(0),3(1),4(2),5(3),6(5)))) e26/26) v1 -- 0x558e83c9ea00 con 0x558e818d7c00 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:03.604+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:03.604+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e8417b500 seq 17 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:03.604841+0000 1 0.000392 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [3,0,5,4,1,2] newacting [3,0,5,4,1,2147483647] 2019-12-07T19:34:03.604+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:03.604+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:03.604+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]p3(0) r=1 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.604+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:03.604+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] state: transitioning to Primary 2019-12-07T19:34:03.604+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417b500 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:03.604+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417b500 onodes shared_blobs 2019-12-07T19:34:03.604+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417b500 allocated 0x[] released 0x[] 2019-12-07T19:34:03.604+0000 7fe2c8267700 15 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:03.604870+0000 0 0.000000 2019-12-07T19:34:03.604+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 prepare 2019-12-07T19:34:03.604+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417b500 2019-12-07T19:34:03.604+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 io_done 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] enter Started/Primary 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] enter Started/Primary/Peering 2019-12-07T19:34:03.604+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(25-25 up [3,0,5,4,1,2](3) acting [3,0,5,4,1,2](3)) up_thru 24 up_from 13 last_epoch_clean 19 interval(25-25 up [3,0,5,4,1,2](3) acting [3,0,5,4,1,2](3)) : primary up 13-24 does not include interval 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] enter Started/Primary/Peering/GetInfo 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] noting past ([18,25] all_participants=0(1),1(4),2(5),3(0),4(3),5(2),6(5) intervals=([18,23] acting 0(1),1(4),3(0),4(3),5(2),6(5))) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] build_prior all_probe 0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] build_prior maybe_rw interval:18, acting: 0(0),1(2),3(5),4(1),5(4),6(3) 2019-12-07T19:34:03.604+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] build_prior prior osd.6 is down 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d1570] 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 27.059623341s + 0s) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] build_prior final: probe 0(0),1(2),3(5),4(1),5(3),5(4),7(4) down 6 blocked_by {} 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] up_thru 25 < same_since 26, must notify monitor 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: querying info from osd.1(2) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: querying info from osd.3(5) 2019-12-07T19:34:03.604+0000 7fe2c8267700 1 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval up [3,0,5,4,1,2] -> [3,0,5,4,1,2], acting [3,0,5,4,1,2] -> [3,0,5,4,1,2147483647], acting_primary 3(0) -> 3, up_primary 3(0) -> 3, role 1 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: querying info from osd.4(1) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: querying info from osd.5(3) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: querying info from osd.5(4) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: querying info from osd.7(4) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] check_prior_readable_down_osds prior_readable_down_osds osd.6 is dead as of epoch 26 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] empty prior_readable_down_osds, clearing ub 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:03.604+0000 7fe2c6a64700 15 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] publish_stats_to_osd 25:38 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 25 while current is 26 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 4(1) notify: (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] got osd.4(1) 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:17.043233+0000, must = 0 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: Adding osd: 4 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.2s2 1.2s2 query(info 0'0 epoch_sent 26) e26/26) v1 -- 0x558e8417cc80 con 0x558e818d7c00 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 26 send_incremental_map 25 -> 26 to 0x558e818d7000 v2:172.21.15.131:6812/37076 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(26..26 src has 1..26) v4 -- 0x558e81222900 con 0x558e818d7000 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 26 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 25 -> 26 (shared) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_query2(1.2s5 1.2s5 query(info 0'0 epoch_sent 26) e26/26) v1 -- 0x558e8417cf00 con 0x558e818d7000 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 26 send_incremental_map 25 -> 26 to 0x558e818fe400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(26..26 src has 1..26) v4 -- 0x558e83d47200 con 0x558e818fe400 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 26 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 25 -> 26 (shared) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_query2(1.2s1 1.2s1 query(info 0'0 epoch_sent 26) e26/26) v1 -- 0x558e8417d180 con 0x558e818fe400 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 26 send_incremental_map 25 -> 26 to 0x558e818d6800 v2:172.21.15.109:6802/36936 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(26..26 src has 1..26) v4 -- 0x558e8405d200 con 0x558e818d6800 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 26 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 25 -> 26 (shared) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.2s3 1.2s3 query(info 0'0 epoch_sent 26) e26/26) v1 -- 0x558e8417d400 con 0x558e818d6800 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 26) e26/26) v1 -- 0x558e8417d680 con 0x558e818d6800 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 26 send_incremental_map 25 -> 26 to 0x558e818fe000 v2:172.21.15.109:6826/36939 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(26..26 src has 1..26) v4 -- 0x558e8405d440 con 0x558e818fe000 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 26 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 25 -> 26 (shared) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 26) e26/26) v1 -- 0x558e8417c500 con 0x558e818fe000 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e81a0b800 seq 23 2019-12-07T19:34:03.604+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b800 cost 5467 (1 ios * 4000 + 1467 bytes) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b800 onodes shared_blobs 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b800 allocated 0x[] released 0x[] 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 prepare 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b800 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 io_done 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 26 queue_want_up_thru now 26 (was 25), currently 25 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 26 send_alive up_thru currently 25 want 26 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 26 send_alive want 26 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- osd_alive(want up_thru 26 have 26) v1 -- 0x558e819b14a0 con 0x558e80f39000 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558e8193b000 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.as3 25 -> 26 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 25 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647] -- 1/1 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [1,4,2,0,5,3] newacting [1,2147483647,2,0,5,2147483647] 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:03.605315+0000 3 0.000068 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:03.605325+0000 0 0.000000 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.604+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 22 ==== osd_map(25..26 src has 1..26) v4 ==== 1263+0+0 (crc 0 0 0) 0x558e83ff1b00 con 0x558e80f39000 2019-12-07T19:34:03.604+0000 7fe2d959f700 20 osd.0 26 OSD::ms_dispatch: osd_map(25..26 src has 1..26) v4 2019-12-07T19:34:03.604+0000 7fe2d959f700 10 osd.0 26 do_waiters -- start 2019-12-07T19:34:03.604+0000 7fe2d959f700 10 osd.0 26 do_waiters -- finish 2019-12-07T19:34:03.604+0000 7fe2d959f700 20 osd.0 26 _dispatch 0x558e83ff1b00 osd_map(25..26 src has 1..26) v4 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:03.604+0000 7fe2d959f700 3 osd.0 26 handle_osd_map epochs [25,26], i have 26, src has [1,26] 2019-12-07T19:34:03.604+0000 7fe2d959f700 10 osd.0 26 no new maps here, dropping 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [1,4,2,0,5,3] newacting [1,2147483647,2,0,5,2147483647] 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.604+0000 7fe2c6a64700 15 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'2 unknown NOTIFY mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(25-25 up [1,4,2,0,5,3](1) acting [1,4,2,0,5,3](1)) up_thru 24 up_from 13 last_epoch_clean 19 interval(25-25 up [1,4,2,0,5,3](1) acting [1,4,2,0,5,3](1)) : primary up 13-24 does not include interval 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] noting past ([18,25] all_participants=0(3),1(0),2(2),3(5),4(1),4(5),5(4),6(1) intervals=([18,23] acting 0(3),1(0),2(2),4(5),5(4),6(1))) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 27.060140244s + 0s) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval up [1,4,2,0,5,3] -> [1,4,2,0,5,3], acting [1,4,2,0,5,3] -> [1,2147483647,2,0,5,2147483647], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 3 -> 3, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] release_backoffs [1:50000000::::0,1:60000000::::head) 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:03.605589+0000 1 0.000273 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:03.605619+0000 0 0.000000 2019-12-07T19:34:03.604+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:03.604+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 25 while current is 26 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.604+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:03.604+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.as0 1.as0 (query:26 sent:26 1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 3->0 ([18,25] all_participants=0(3),1(0),2(2),3(5),4(1),4(5),5(4),6(1) intervals=([18,23] acting 0(3),1(0),2(2),4(5),5(4),6(1)))) e26/26) v1 -- 0x558e83c49100 con 0x558e818d7c00 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e81a0a600 seq 13 2019-12-07T19:34:03.605+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a600 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a600 onodes shared_blobs 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a600 allocated 0x[] released 0x[] 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 prepare 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a600 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 io_done 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558e81075000 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 26 queue_want_up_thru want 26 <= queued 26, currently 25 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e26) queued 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e26) pg 0x558e81075000 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 26 dequeue_op 0x558e819431e0 prio 63 cost 4 latency 1.951470 osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 26 send_incremental_map 25 -> 26 to 0x558e83d68000 172.21.15.131:0/224485236 2019-12-07T19:34:03.605+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_map(26..26 src has 1..26) v4 -- 0x558e8405d680 con 0x558e83d68000 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 26 maybe_share_map con 0x558e83d68000 172.21.15.131:0/224485236 map epoch 25 -> 26 (shared) 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 26 dequeue_op 0x558e819431e0 finish 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e26) queued 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e26) pg 0x558e81075000 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 26 dequeue_op 0x558e81943080 prio 63 cost 4 latency 1.951432 osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 26 dequeue_op 0x558e81943080 finish 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e26) queued 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e26) pg 0x558e81075000 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 26 dequeue_op 0x558e819438c0 prio 63 cost 4 latency 1.951425 osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 26 dequeue_op 0x558e819438c0 finish 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558e8192b000 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.9s3 25 -> 26 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,2147483647] -- 1/1 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,3] newacting [1,4,5,0,7,2147483647] 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:03.605+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 26 ==== pg_notify((query:26 sent:26 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 26) v7 ==== 985+0+0 (unknown 0 0 0) 0x558e8403e780 con 0x558e818d6800 2019-12-07T19:34:03.605+0000 7fe2e753b700 7 osd.0 26 handle_fast_pg_notify pg_notify((query:26 sent:26 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) epoch 26) v7 from osd.5 2019-12-07T19:34:03.605+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:03.605907+0000 3 0.000084 2019-12-07T19:34:03.605+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:03.605918+0000 0 0.000000 2019-12-07T19:34:03.605+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:03.605+0000 7fe2e753b700 15 osd.0 26 enqueue_peering_evt 1.2s0 epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(3) notify: (query:26 sent:26 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.605+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(3) notify: (query:26 sent:26 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=25 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,3] newacting [1,4,5,0,7,2147483647] 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.605+0000 7fe2cb26d700 15 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,25)/1 crt=21'6 unknown NOTIFY mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(25-25 up [1,4,5,0,7,3](1) acting [1,4,5,0,7,3](1)) up_thru 24 up_from 13 last_epoch_clean 19 interval(25-25 up [1,4,5,0,7,3](1) acting [1,4,5,0,7,3](1)) : primary up 13-24 does not include interval 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] noting past ([18,25] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4))) 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 27.060730930s + 0s) 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:03.605+0000 7fe2cb26d700 1 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval up [1,4,5,0,7,3] -> [1,4,5,0,7,3], acting [1,4,5,0,7,3] -> [1,4,5,0,7,2147483647], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 3 -> 3, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] release_backoffs [1:90000000::::0,1:a0000000::::head) 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:03.605+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 27 ==== osd_map(26..26 src has 1..26) v4 ==== 913+0+0 (unknown 0 0 0) 0x558e8405d440 con 0x558e818d6800 2019-12-07T19:34:03.605+0000 7fe2d4a80700 20 osd.0 26 OSD::ms_dispatch: osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.605+0000 7fe2d4a80700 10 osd.0 26 do_waiters -- start 2019-12-07T19:34:03.605+0000 7fe2d4a80700 10 osd.0 26 do_waiters -- finish 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:03.605+0000 7fe2d4a80700 20 osd.0 26 _dispatch 0x558e8405d440 osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.605+0000 7fe2d4a80700 3 osd.0 26 handle_osd_map epochs [26,26], i have 26, src has [1,26] 2019-12-07T19:34:03.605+0000 7fe2d4a80700 10 osd.0 26 no new maps here, dropping 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(3) notify: (query:26 sent:26 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26)> waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(3) notify: (query:26 sent:26 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) pg 0x558e81075000 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(3) notify: (query:26 sent:26 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 3->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.605+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:03.606178+0000 1 0.000269 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] got osd.5(3) 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:03.605+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:03.605+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:03.778478+0000, must = 0 2019-12-07T19:34:03.605+0000 7fe2cb26d700 1 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:03.605+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:03.606210+0000 0 0.000000 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:03.605+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:03.605+0000 7fe2c6a64700 10 osd.0 26 queue_want_up_thru want 26 <= queued 26, currently 25 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 25 while current is 26 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:03.605+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:26 sent:26 1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 3->0 ([18,25] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4)))) e26/26) v1 -- 0x558e83c9d500 con 0x558e818d7c00 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83d4c000 seq 17 2019-12-07T19:34:03.605+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c000 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c000 onodes shared_blobs 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c000 allocated 0x[] released 0x[] 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 prepare 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c000 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 io_done 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558e819e4000 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.1s1 25 -> 26 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 25 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map [3,0,2,4,5,1]/[3,0,2147483647,4,5,1] -- 3/3 2019-12-07T19:34:03.605+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 15 ==== pg_notify((query:26 sent:26 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 26) v7 ==== 985+0+0 (unknown 0 0 0) 0x558e8403e780 con 0x558e818fe000 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.605+0000 7fe2e753b700 7 osd.0 26 handle_fast_pg_notify pg_notify((query:26 sent:26 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 26) v7 from osd.7 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [3,0,2,4,5,1] newacting [3,0,2147483647,4,5,1] 2019-12-07T19:34:03.605+0000 7fe2e753b700 15 osd.0 26 enqueue_peering_evt 1.2s0 epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 7(4) notify: (query:26 sent:26 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:03.605+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 7(4) notify: (query:26 sent:26 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) 2019-12-07T19:34:03.605+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:03.606392+0000 3 0.000084 2019-12-07T19:34:03.605+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:03.606402+0000 0 0.000000 2019-12-07T19:34:03.605+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=25 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=26 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=26 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=26 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=26 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [3,0,2,4,5,1] newacting [3,0,2147483647,4,5,1] 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=26 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]p3(0) r=1 lpr=26 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.605+0000 7fe2cb26d700 15 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,25)/1 crt=21'3 unknown NOTIFY mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:03.605+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 16 ==== osd_map(26..26 src has 1..26) v4 ==== 913+0+0 (unknown 0 0 0) 0x558e8405d200 con 0x558e818fe000 2019-12-07T19:34:03.605+0000 7fe2d4a80700 20 osd.0 26 OSD::ms_dispatch: osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.605+0000 7fe2d4a80700 10 osd.0 26 do_waiters -- start 2019-12-07T19:34:03.605+0000 7fe2d4a80700 10 osd.0 26 do_waiters -- finish 2019-12-07T19:34:03.605+0000 7fe2d4a80700 20 osd.0 26 _dispatch 0x558e8405d200 osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.605+0000 7fe2d4a80700 3 osd.0 26 handle_osd_map epochs [26,26], i have 26, src has [1,26] 2019-12-07T19:34:03.605+0000 7fe2d4a80700 10 osd.0 26 no new maps here, dropping 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(25-25 up [3,0,2,4,5,1](3) acting [3,0,2,4,5,1](3)) up_thru 24 up_from 13 last_epoch_clean 19 interval(25-25 up [3,0,2,4,5,1](3) acting [3,0,2,4,5,1](3)) : primary up 13-24 does not include interval 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] noting past ([18,25] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),6(2) intervals=([18,23] acting 0(1),1(5),3(0),4(3),5(4),6(2))) 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d1570] 2019-12-07T19:34:03.605+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 27.061214130s + 0s) 2019-12-07T19:34:03.605+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:03.605+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:03.605+0000 7fe2cb26d700 1 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval up [3,0,2,4,5,1] -> [3,0,2,4,5,1], acting [3,0,2,4,5,1] -> [3,0,2147483647,4,5,1], acting_primary 3(0) -> 3, up_primary 3(0) -> 3, role 1 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:03.605+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:03.605+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 7(4) notify: (query:26 sent:26 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.605+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26)> waiting <> waiting_peering {} 2019-12-07T19:34:03.605+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 7(4) notify: (query:26 sent:26 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) pg 0x558e81075000 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] release_backoffs [1:80000000::::0,1:90000000::::head) 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:03.605+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 7(4) notify: (query:26 sent:26 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:03.605+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] got osd.7(4) 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:03.605+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:06.072918+0000, must = 0 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:03.605+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.605+0000 7fe2caa6c700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: Adding osd: 7 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:03.605+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:03.605+0000 7fe2caa6c700 10 osd.0 26 queue_want_up_thru want 26 <= queued 26, currently 25 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:03.605+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.605+0000 7fe2cb26d700 20 osd.0 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.605+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.605+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:03.606666+0000 1 0.000271 2019-12-07T19:34:03.605+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:03.603+0000 7fe2c5261700 10 osd.0 pg_epoch: 26 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.605+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:03.606+0000 7fe2cb26d700 1 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:03.606+0000 7fe2c5261700 10 osd.0 pg_epoch: 26 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.606+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:03.606699+0000 0 0.000000 2019-12-07T19:34:03.606+0000 7fe2c5261700 20 osd.0 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.606+0000 7fe2cb26d700 5 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:03.606+0000 7fe2c5261700 10 osd.0 pg_epoch: 26 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.606+0000 7fe2cb26d700 20 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 25 while current is 26 2019-12-07T19:34:03.606+0000 7fe2c5261700 10 osd.0 pg_epoch: 26 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.606+0000 7fe2c5261700 20 osd.0 pg_epoch: 26 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 26 2019-12-07T19:34:03.606+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:03.606+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:03.606+0000 7fe2c5261700 10 osd.0 pg_epoch: 26 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.606+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:03.606+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:03.606+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.606+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:03.606+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.1s0 1.1s0 (query:26 sent:26 1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),6(2) intervals=([18,23] acting 0(1),1(5),3(0),4(3),5(4),6(2)))) e26/26) v1 -- 0x558e8404ae00 con 0x558e818d7000 2019-12-07T19:34:03.606+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:03.606+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e83d4c300 seq 15 2019-12-07T19:34:03.606+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:03.606+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:03.606+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:03.606+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:03.606+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:03.606+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c300 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:03.606+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c300 onodes shared_blobs 2019-12-07T19:34:03.606+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c300 allocated 0x[] released 0x[] 2019-12-07T19:34:03.606+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 prepare 2019-12-07T19:34:03.606+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c300 2019-12-07T19:34:03.606+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 io_done 2019-12-07T19:34:03.606+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:03.606+0000 7fe2cc26f700 20 osd.0 26 agent_entry empty queue 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 6 submitting 6 deferred done 1 stable 0 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2571 < min_flush_size 524288 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3882 < min_flush_size 524288 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5173 < min_flush_size 524288 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5267 < min_flush_size 524288 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 6464 < min_flush_size 524288 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 7744 < min_flush_size 524288 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 7767 < min_flush_size 524288 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 7767 < min_flush_size 524288 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x76b6d~1e57 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x76b6d 0x76b6d~1e57 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x46b6d 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb6d 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x9c4 and padding block with 0x63c 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x789c4 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:03.606+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 6 cleaned 0 in 0.000193619s (0.000000089s flush + 0.000193530s kv commit) 2019-12-07T19:34:03.606+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1b00,0x558e8417b500,0x558e81a0b800,0x558e81a0a600,0x558e83d4c000,0x558e83d4c300> 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 kv_submitted 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1b00 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 finishing 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1b00 onodes 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1b00 done 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1b00 [] 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 kv_submitted 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417b500 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 finishing 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417b500 onodes 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b500 done 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417b500 [] 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 kv_submitted 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b800 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 finishing 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b800 onodes 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 done 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b800 [] 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 kv_submitted 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a600 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 finishing 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a600 onodes 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a600 done 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a600 [] 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 kv_submitted 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c000 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 finishing 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c000 onodes 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c000 done 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c000 [] 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 kv_submitted 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c300 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 finishing 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c300 onodes 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c300 done 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c300 [] 2019-12-07T19:34:03.606+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:03.606+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:03.606+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:03.606+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:03.606+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:03.606+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:03.606+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:03.606+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:03.607+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 66 ==== pg_notify2(1.2s0 1.2s0 (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e26/26) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e80ff3100 con 0x558e818fe400 2019-12-07T19:34:03.607+0000 7fe2e6d3a700 15 osd.0 26 enqueue_peering_evt 1.2s0 epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 4(1) notify: (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.607+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 4(1) notify: (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) 2019-12-07T19:34:03.607+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 28 ==== pg_notify2(1.2s0 1.2s0 (query:26 sent:26 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e26/26) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e8122b800 con 0x558e818d6800 2019-12-07T19:34:03.607+0000 7fe2e753b700 15 osd.0 26 enqueue_peering_evt 1.2s0 epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(4) notify: (query:26 sent:26 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.607+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(4) notify: (query:26 sent:26 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) 2019-12-07T19:34:03.607+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 29 ==== pg_notify2(1.2s0 1.2s0 (query:26 sent:26 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e26/26) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e8122b800 con 0x558e818d6800 2019-12-07T19:34:03.607+0000 7fe2e753b700 15 osd.0 26 enqueue_peering_evt 1.2s0 epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(4) notify: (query:26 sent:26 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.607+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(4) notify: (query:26 sent:26 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) 2019-12-07T19:34:03.607+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.607+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 4(1) notify: (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.607+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26)> waiting <> waiting_peering {} 2019-12-07T19:34:03.607+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 4(1) notify: (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) pg 0x558e81075000 2019-12-07T19:34:03.607+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 23 ==== pg_query2(1.5s4 1.5s4 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417c500 con 0x558e818d6000 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 4(1) notify: (query:26 sent:26 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.607+0000 7fe2e753b700 15 osd.0 26 enqueue_peering_evt 1.5s4 epoch_sent: 26 epoch_requested: 26 MQuery 1.5s4 from 2(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.607+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.5s4 from 2(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] got dup osd.4(1) info 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26), identical to ours 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 osd.0 26 queue_want_up_thru want 26 <= queued 26, currently 25 2019-12-07T19:34:03.607+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.607+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(4) notify: (query:26 sent:26 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.607+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.607+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26)> waiting <> waiting_peering {} 2019-12-07T19:34:03.607+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.5s4 from 2(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.607+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(4) notify: (query:26 sent:26 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) pg 0x558e81075000 2019-12-07T19:34:03.607+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.607+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.5s4 from 2(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0 2019-12-07T19:34:03.607+0000 7fe2c9269700 10 osd.0 26 handle_pg_query_nopg 1.5s4 2019-12-07T19:34:03.607+0000 7fe2c9269700 10 osd.0 26 pg 1.5s4 dne 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(4) notify: (query:26 sent:26 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.607+0000 7fe2c9269700 10 osd.0 26 send_incremental_map 25 -> 26 to 0x558e818d6000 v2:172.21.15.131:6804/37077 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] got osd.5(4) 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:03.607+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(26..26 src has 1..26) v4 -- 0x558e81957200 con 0x558e818d6000 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:01.147575+0000, must = 0 2019-12-07T19:34:03.607+0000 7fe2c9269700 10 osd.0 26 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 25 -> 26 (shared) 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] osd.5(4) has stray content: 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:03.607+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify((query:26 sent:26 1.5s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 4->0 ([0,0] all_participants= intervals=)) epoch 26) v7 -- 0x558e8403fc20 con 0x558e818d6000 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:03.607+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:03.607+0000 7fe2caa6c700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 osd.0 26 queue_want_up_thru want 26 <= queued 26, currently 25 2019-12-07T19:34:03.607+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.607+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(4) notify: (query:26 sent:26 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.607+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26)> waiting <> waiting_peering {} 2019-12-07T19:34:03.607+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(4) notify: (query:26 sent:26 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) pg 0x558e81075000 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 5(4) notify: (query:26 sent:26 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] got dup osd.5(4) info 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26), identical to ours 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:03.607+0000 7fe2caa6c700 10 osd.0 26 queue_want_up_thru want 26 <= queued 26, currently 25 2019-12-07T19:34:03.607+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:03.607+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 24 ==== osd_map(26..26 src has 1..26) v4 ==== 913+0+0 (unknown 0 0 0) 0x558e817df680 con 0x558e818d6000 2019-12-07T19:34:03.607+0000 7fe2d4a80700 20 osd.0 26 OSD::ms_dispatch: osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.607+0000 7fe2d4a80700 10 osd.0 26 do_waiters -- start 2019-12-07T19:34:03.607+0000 7fe2d4a80700 10 osd.0 26 do_waiters -- finish 2019-12-07T19:34:03.607+0000 7fe2d4a80700 20 osd.0 26 _dispatch 0x558e817df680 osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.607+0000 7fe2d4a80700 3 osd.0 26 handle_osd_map epochs [26,26], i have 26, src has [1,26] 2019-12-07T19:34:03.607+0000 7fe2d4a80700 10 osd.0 26 no new maps here, dropping 2019-12-07T19:34:03.607+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 61 ==== pg_query2(1.1s1 1.1s1 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417c000 con 0x558e818d7000 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.1s1 epoch_sent: 26 epoch_requested: 26 MQuery 1.1s1 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.1s1 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.607+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.607+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.1s1 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 62 ==== pg_notify2(1.2s0 1.2s0 (query:26 sent:26 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e26/26) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e83c9dc00 con 0x558e818d7000 2019-12-07T19:34:03.607+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.607+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.1s1 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0x558e819e4000 2019-12-07T19:34:03.607+0000 7fe2c7265700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MQuery 1.1s1 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.2s0 epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 3(5) notify: (query:26 sent:26 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 3(5) notify: (query:26 sent:26 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) 2019-12-07T19:34:03.607+0000 7fe2c7265700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:03.607+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:03.607+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.1s0 1.1s0 (query:26 sent:26 1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),6(2) intervals=([18,23] acting 0(1),1(5),3(0),4(3),5(4),6(2)))) e26/26) v1 -- 0x558e80ff3800 con 0x558e818d7000 2019-12-07T19:34:03.607+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 23 ==== osd_map(26..26 src has 1..26) v4 ==== 913+0+0 (crc 0 0 0) 0x558e83f2a480 con 0x558e80f39000 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 64 ==== pg_notify2(1.2s0 1.2s0 (query:26 sent:26 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e26/26) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e83c9dc00 con 0x558e818d7000 2019-12-07T19:34:03.607+0000 7fe2d959f700 20 osd.0 26 OSD::ms_dispatch: osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.607+0000 7fe2d959f700 10 osd.0 26 do_waiters -- start 2019-12-07T19:34:03.607+0000 7fe2d959f700 10 osd.0 26 do_waiters -- finish 2019-12-07T19:34:03.607+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:03.607+0000 7fe2d959f700 20 osd.0 26 _dispatch 0x558e83f2a480 osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.607+0000 7fe2d959f700 3 osd.0 26 handle_osd_map epochs [26,26], i have 26, src has [1,26] 2019-12-07T19:34:03.607+0000 7fe2d959f700 10 osd.0 26 no new maps here, dropping 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.2s0 epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 3(5) notify: (query:26 sent:26 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 3(5) notify: (query:26 sent:26 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 109 ==== pg_query2(1.9s3 1.9s3 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417cf00 con 0x558e818d7c00 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.9s3 epoch_sent: 26 epoch_requested: 26 MQuery 1.9s3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.9s3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 110 ==== pg_query2(1.as3 1.as3 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417cc80 con 0x558e818d7c00 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.as3 epoch_sent: 26 epoch_requested: 26 MQuery 1.as3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.as3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.607+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 111 ==== pg_notify2(1.2s0 1.2s0 (query:26 sent:26 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e26/26) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e83c9ea00 con 0x558e818d7c00 2019-12-07T19:34:03.607+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0x558e81928000 2019-12-07T19:34:03.608+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.2s0 epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 1(2) notify: (query:26 sent:26 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.608+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 1(2) notify: (query:26 sent:26 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map: 26 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0:3.update_pg_epoch 1.bs3 25 -> 26 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 25 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5] -- 4/4 2019-12-07T19:34:03.608+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 113 ==== pg_notify2(1.2s0 1.2s0 (query:26 sent:26 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e26/26) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e83c9ea00 con 0x558e818d7c00 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [4,0,2,7,1,5] newacting [4,2147483647,2,2147483647,1,5] 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:03.608+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.2s0 epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 1(2) notify: (query:26 sent:26 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.608+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 1(2) notify: (query:26 sent:26 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) 2019-12-07T19:34:03.608+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:03.608760+0000 3 0.000067 2019-12-07T19:34:03.608+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 114 ==== pg_query2(1.es4 1.es4 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417cc80 con 0x558e818d7c00 2019-12-07T19:34:03.608+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:03.608770+0000 0 0.000000 2019-12-07T19:34:03.608+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.es4 epoch_sent: 26 epoch_requested: 26 MQuery 1.es4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.608+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.es4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.608+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=25 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=26 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=26 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=26 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=26 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] new interval newup [4,0,2,7,1,5] newacting [4,2147483647,2,2147483647,1,5] 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=26 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]p4(0) r=-1 lpr=26 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] set_last_peering_reset 26 2019-12-07T19:34:03.608+0000 7fe2c6263700 15 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,25)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(25-25 up [4,0,2,7,1,5](4) acting [4,0,2,7,1,5](4)) up_thru 24 up_from 13 last_epoch_clean 19 interval(25-25 up [4,0,2,7,1,5](4) acting [4,0,2,7,1,5](4)) : primary up 13-24 does not include interval 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=25) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] noting past ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] init_hb_stamps now [] 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 27.063582989s + 0s) 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:03.608+0000 7fe2c6263700 1 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] start_peering_interval up [4,0,2,7,1,5] -> [4,0,2,7,1,5], acting [4,0,2,7,1,5] -> [4,2147483647,2,2147483647,1,5], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role -1 -> -1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] release_backoffs [1:d0000000::::0,1:e0000000::::head) 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.608+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:03.608984+0000 1 0.000214 2019-12-07T19:34:03.608+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:03.608+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:03.608+0000 7fe2c6263700 1 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:03.608+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:03.609001+0000 0 0.000000 2019-12-07T19:34:03.608+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 25 while current is 26 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MQuery 1.bs3 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:03.608+0000 7fe2c6263700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:26 sent:26 1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 3->0 ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e26/26) v1 -- 0x558e83eba000 con 0x558e818fe400 2019-12-07T19:34:03.608+0000 7fe2c6263700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:26 sent:26 1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 3->0 ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e26/26) v1 -- 0x558e83eba700 con 0x558e818fe400 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e8195e600 seq 27 2019-12-07T19:34:03.608+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e600 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e600 onodes shared_blobs 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e600 allocated 0x[] released 0x[] 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 prepare 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e600 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 io_done 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs1 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.bs1 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 26 handle_pg_query_nopg 1.bs1 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 26 pg 1.bs1 dne 2019-12-07T19:34:03.608+0000 7fe2c6263700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify((query:26 sent:26 1.bs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 1->0 ([0,0] all_participants= intervals=)) epoch 26) v7 -- 0x558e8403fa40 con 0x558e818fe400 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.3s5 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.3s5 from 4(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 26 handle_pg_query_nopg 1.3s5 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 26 pg 1.3s5 dne 2019-12-07T19:34:03.608+0000 7fe2c6263700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify((query:26 sent:26 1.3s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 26) v7 -- 0x558e84198000 con 0x558e818fe400 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 NullEvt) prio 255 cost 10 e26) pg 0x558e81928000 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.608+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:03.608+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:03.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:03.604+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:03.608+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 63 ==== osd_map(26..26 src has 1..26) v4 ==== 913+0+0 (unknown 0 0 0) 0x558e81222900 con 0x558e818d7000 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] release_backoffs [1:f0000000::::0,MAX) 2019-12-07T19:34:03.608+0000 7fe2d4a80700 20 osd.0 26 OSD::ms_dispatch: osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.608+0000 7fe2d4a80700 10 osd.0 26 do_waiters -- start 2019-12-07T19:34:03.608+0000 7fe2d4a80700 10 osd.0 26 do_waiters -- finish 2019-12-07T19:34:03.608+0000 7fe2d4a80700 20 osd.0 26 _dispatch 0x558e81222900 osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.608+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.608+0000 7fe2d4a80700 3 osd.0 26 handle_osd_map epochs [26,26], i have 26, src has [1,26] 2019-12-07T19:34:03.608+0000 7fe2d4a80700 10 osd.0 26 no new maps here, dropping 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:03.608+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 112 ==== osd_map(26..26 src has 1..26) v4 ==== 913+0+0 (unknown 0 0 0) 0x558e83d46240 con 0x558e818d7c00 2019-12-07T19:34:03.608+0000 7fe2d4a80700 20 osd.0 26 OSD::ms_dispatch: osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.608+0000 7fe2d4a80700 10 osd.0 26 do_waiters -- start 2019-12-07T19:34:03.608+0000 7fe2d4a80700 10 osd.0 26 do_waiters -- finish 2019-12-07T19:34:03.608+0000 7fe2d4a80700 20 osd.0 26 _dispatch 0x558e83d46240 osd_map(26..26 src has 1..26) v4 2019-12-07T19:34:03.608+0000 7fe2d4a80700 3 osd.0 26 handle_osd_map epochs [26,26], i have 26, src has [1,26] 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:03.608+0000 7fe2d4a80700 10 osd.0 26 no new maps here, dropping 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:03.608+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 65 ==== pg_query2(1.fs1 1.fs1 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417cc80 con 0x558e818d7000 2019-12-07T19:34:03.608+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.fs1 epoch_sent: 26 epoch_requested: 26 MQuery 1.fs1 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.608+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.fs1 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] on_change 2019-12-07T19:34:03.608+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:03.608+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:03.608+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:03.608+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.fs1 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.608+0000 7fe2c8267700 20 osd.0 26 advance_pg new pool opts old pool opts 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 26 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:03.608+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:03.609322+0000 1 0.004531 2019-12-07T19:34:03.608+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:03.608+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:03.608+0000 7fe2c8267700 1 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:03.608+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:03.609339+0000 0 0.000000 2019-12-07T19:34:03.608+0000 7fe2c8267700 5 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:03.608+0000 7fe2c8267700 20 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 25 while current is 26 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 NullEvt 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:03.608+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.fs0 1.fs0 (query:26 sent:26 1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(1),1(4),2(5),3(0),4(3),5(2),6(5) intervals=([18,23] acting 0(1),1(4),3(0),4(3),5(2),6(5)))) e26/26) v1 -- 0x558e8404c300 con 0x558e818d7000 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:03.608+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e819a0000 seq 21 2019-12-07T19:34:03.608+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:03.608+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:03.608+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:34:03.608+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0000 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:03.608+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0000 onodes shared_blobs 2019-12-07T19:34:03.608+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0000 allocated 0x[] released 0x[] 2019-12-07T19:34:03.608+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.608+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.9s3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.608+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.608+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.9s3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0x558e8192b000 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 prepare 2019-12-07T19:34:03.608+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0000 2019-12-07T19:34:03.608+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 io_done 2019-12-07T19:34:03.608+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MQuery 1.9s3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.608+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:03.608+0000 7fe2cb26d700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:03.608+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.608+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.fs1 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0x558e81a9b000 2019-12-07T19:34:03.608+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:03.608+0000 7fe2c425f700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MQuery 1.fs1 from 3(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.608+0000 7fe2c425f700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:03.608+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:26 sent:26 1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 3->0 ([18,25] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4)))) e26/26) v1 -- 0x558e8404b500 con 0x558e818d7c00 2019-12-07T19:34:03.608+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:03.608+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:03.608+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.fs0 1.fs0 (query:26 sent:26 1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(1),1(4),2(5),3(0),4(3),5(2),6(5) intervals=([18,23] acting 0(1),1(4),3(0),4(3),5(2),6(5)))) e26/26) v1 -- 0x558e83ebc300 con 0x558e818d7000 2019-12-07T19:34:03.608+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 1 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 3(5) notify: (query:26 sent:26 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26)> waiting <> waiting_peering {} 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 3(5) notify: (query:26 sent:26 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) pg 0x558e81075000 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 3(5) notify: (query:26 sent:26 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] got osd.3(5) 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2571 < min_flush_size 524288 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:01.125382+0000, must = 0 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: Adding osd: 3 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2642 < min_flush_size 524288 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 26 queue_want_up_thru want 26 <= queued 26, currently 25 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2642 < min_flush_size 524288 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 3(5) notify: (query:26 sent:26 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26)> waiting <> waiting_peering {} 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 3(5) notify: (query:26 sent:26 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) pg 0x558e81075000 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x789c4~a52 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 3(5) notify: (query:26 sent:26 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x789c4 0x789c4~a52 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] got dup osd.3(5) info 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26), identical to ours 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 26 queue_want_up_thru want 26 <= queued 26, currently 25 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x489c4 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x9c4 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x416 and padding block with 0xbea 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.as3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.as3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0x558e8193b000 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MQuery 1.as3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:03.609+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.as0 1.as0 (query:26 sent:26 1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 3->0 ([18,25] all_participants=0(3),1(0),2(2),3(5),4(1),4(5),5(4),6(1) intervals=([18,23] acting 0(3),1(0),2(2),4(5),5(4),6(1)))) e26/26) v1 -- 0x558e83c49800 con 0x558e818d7c00 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 1(2) notify: (query:26 sent:26 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26)> waiting <> waiting_peering {} 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 1(2) notify: (query:26 sent:26 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) pg 0x558e81075000 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 1(2) notify: (query:26 sent:26 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] got osd.1(2) 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x79416 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:20.387440+0000, must = 0 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: Adding osd: 1 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: Common peer features: 3f01cfb8ffadffff 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 1 in 0.000130089s (0.000000097s flush + 0.000129992s kv commit) 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: Common acting features: 3f01cfb8ffadffff 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: Common upacting features: 3f01cfb8ffadffff 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e600,0x558e819a0000> 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <0x558e83c6e140> 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 kv_submitted 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e600 2019-12-07T19:34:03.609+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] exit Started/Primary/Peering/GetInfo 2019-12-07T19:34:03.609843+0000 10 0.000412 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 finishing 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e600 onodes 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e600 done 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e600 [] 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 kv_submitted 2019-12-07T19:34:03.609+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] enter Started/Primary/Peering/GetLog 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0000 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 finishing 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0000 onodes 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0000 done 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0000 [] 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 deferred_cleanup 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a900 onodes 0x558e80251b80,0x558e817b9180,0x558e83f28280,0x558e83f28500 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03c18 in 0x558e80219340) _finish_write discard buffer(0x558e84033320 space 0x558e83d03c18 0x0~1000 writing nocache) 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03c88 in 0x558e80219340) _finish_write discard buffer(0x558e840333b0 space 0x558e83d03c88 0x0~1000 writing nocache) 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] choose_acting all_info osd.0(0) 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03d68 in 0x558e80219340) _finish_write discard buffer(0x558e84033440 space 0x558e83d03d68 0x1000~1000 writing nocache) 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a900 done 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0900 deferred_cleanup 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a900 [] 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] choose_acting all_info osd.1(2) 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 deferred_cleanup 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0900 onodes 0x558e80251b80,0x558e817b9180,0x558e8191b400,0x558e83f29900 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] choose_acting all_info osd.3(5) 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af8f68 in 0x558e80219340) _finish_write discard buffer(0x558e81170fc0 space 0x558e83af8f68 0x0~1000 writing nocache) 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0f0b8 in 0x558e80219340) _finish_write discard buffer(0x558e81171ef0 space 0x558e83d0f0b8 0x1000~1000 writing nocache) 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0f748 in 0x558e80219340) _finish_write discard buffer(0x558e81171e60 space 0x558e83d0f748 0x0~1000 writing nocache) 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0900 done 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811faf00 deferred_cleanup 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0900 [] 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] choose_acting all_info osd.4(1) 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 deferred_cleanup 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811faf00 onodes 0x558e80251b80,0x558e817b9180,0x558e83f28780,0x558e83f28a00 2019-12-07T19:34:03.609+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194fcf8 in 0x558e80219340) _finish_write discard buffer(0x558e84033680 space 0x558e8194fcf8 0x0~1000 writing nocache) 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] choose_acting all_info osd.5(3) 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:03.609+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.es4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac5f28 in 0x558e80219340) _finish_write discard buffer(0x558e84033710 space 0x558e83ac5f28 0x1000~1000 writing nocache) 2019-12-07T19:34:03.609+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6f978 in 0x558e80219340) _finish_write discard buffer(0x558e840335f0 space 0x558e83e6f978 0x0~1000 writing nocache) 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811faf00 done 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] choose_acting all_info osd.5(4) 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0300 deferred_cleanup 2019-12-07T19:34:03.609+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.es4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0x558e81a98000 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811faf00 [] 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] choose_acting all_info osd.7(4) 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0300 deferred_cleanup 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0300 onodes 0x558e80251b80,0x558e817b9180,0x558e83f29b80,0x558e8402f680 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _finish_write discard buffer(0x558e83d10090 space 0x558e81078638 0x0~1000 writing nocache) 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _finish_write discard buffer(0x558e83d11b90 space 0x558e81789dd8 0x0~1000 writing nocache) 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f94478 in 0x558e80219340) _finish_write discard buffer(0x558e83d10900 space 0x558e83f94478 0x0~1000 writing nocache) 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] For position 0: selecting up[i]: 0(0) For position 1: selecting up[i]: 4(1) For position 2: selecting up[i]: 1(2) For position 3: selecting up[i]: 5(3) For position 4: selecting up[i]: 7(4) For position 5: selecting up[i]: 3(5) 2019-12-07T19:34:03.609+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MQuery 1.es4 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f94cc8 in 0x558e80219340) _finish_write discard buffer(0x558e83d11c20 space 0x558e83f94cc8 0x1000~1000 writing nocache) 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84014558 in 0x558e80219340) _finish_write discard buffer(0x558e83d119e0 space 0x558e84014558 0x0~1000 writing nocache) 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0300 done 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107a500 q now empty 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0300 [] 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:03.609+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish empty zombie osr 0x558e8107a500 already reaped 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] choose_async_recovery_ec candidates by cost are: 10,5(3),10,7(4) 2019-12-07T19:34:03.609+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:03.609+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:03.609+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] choose_async_recovery_ec result want=[0,4,1,2147483647,2147483647,3] async_recovery=5(3),7(4) 2019-12-07T19:34:03.609+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.es0 1.es0 (query:26 sent:26 1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),6(5) intervals=([18,23] acting 0(4),1(0),3(1),4(2),5(3),6(5)))) e26/26) v1 -- 0x558e83c9f800 con 0x558e818d7c00 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] acting_recovery_backfill is 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:03.609+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] choose_acting want=[0,4,1,2147483647,2147483647,3] backfill_targets= async_recovery_targets=5(3),7(4) 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: leaving GetLog 2019-12-07T19:34:03.609+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] exit Started/Primary/Peering/GetLog 2019-12-07T19:34:03.609994+0000 0 0.000000 2019-12-07T19:34:03.609+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] enter Started/Primary/Peering/GetMissing 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: osd.1(2) has no missing, identical log 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: osd.3(5) has no missing, identical log 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: osd.4(1) has no missing, identical log 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: still need up_thru update before going active 2019-12-07T19:34:03.609+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] exit Started/Primary/Peering/GetMissing 2019-12-07T19:34:03.610023+0000 0 0.000000 2019-12-07T19:34:03.609+0000 7fe2c6a64700 5 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] enter Started/Primary/Peering/WaitUpThru 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e811fa600 seq 24 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa600 cost 4032 (1 ios * 4000 + 32 bytes) 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa600 onodes shared_blobs 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa600 allocated 0x[] released 0x[] 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 prepare 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa600 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 io_done 2019-12-07T19:34:03.609+0000 7fe2c6a64700 10 osd.0 26 queue_want_up_thru want 26 <= queued 26, currently 25 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:03.609+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.609+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 1(2) notify: (query:26 sent:26 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:03.609+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26)> waiting <> waiting_peering {} 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:03.609+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 1(2) notify: (query:26 sent:26 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e26) pg 0x558e81075000 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x79416~2e to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x79416 0x79416~2e to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.609+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MNotifyRec 1.2s0 from 1(2) notify: (query:26 sent:26 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x49416 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x416 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x444 and padding block with 0xbbc 2019-12-07T19:34:03.609+0000 7fe2caa6c700 7 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: handle_pg_notify from osd.1(2) 2019-12-07T19:34:03.609+0000 7fe2caa6c700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] got dup osd.1(2) info 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26), identical to ours 2019-12-07T19:34:03.609+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:03.609+0000 7fe2caa6c700 10 osd.0 26 queue_want_up_thru want 26 <= queued 26, currently 25 2019-12-07T19:34:03.609+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x79444 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:03.609+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000077139s (0.000000087s flush + 0.000077052s kv commit) 2019-12-07T19:34:03.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa600> 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 kv_submitted 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa600 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 finishing 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa600 onodes 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa600 done 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:03.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa600 [] 2019-12-07T19:34:03.609+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:03.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:03.609+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:03.610+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:03.610+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:03.611+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 115 ==== pg_query2(1.8s5 1.8s5 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417cc80 con 0x558e818d7c00 2019-12-07T19:34:03.611+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.8s5 epoch_sent: 26 epoch_requested: 26 MQuery 1.8s5 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.611+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.8s5 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.611+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 116 ==== pg_query2(1.7s3 1.7s3 query(info 0'0 epoch_sent 26) e26/26) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417cc80 con 0x558e818d7c00 2019-12-07T19:34:03.611+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.7s3 epoch_sent: 26 epoch_requested: 26 MQuery 1.7s3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.611+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.7s3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) 2019-12-07T19:34:03.612+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.612+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.7s3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.612+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.612+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.7s3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0x558e8122e000 2019-12-07T19:34:03.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 MQuery 1.7s3 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26) 2019-12-07T19:34:03.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:03.612+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:03.612+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:26 sent:26 1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 3->0 ([18,25] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e26/26) v1 -- 0x558e8404a000 con 0x558e818d7c00 2019-12-07T19:34:03.612+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:03.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 (new) to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:03.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.8s5 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) queued 2019-12-07T19:34:03.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:03.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 MQuery 1.8s5 from 1(0) query_epoch 26 query: query(info 0'0 epoch_sent 26)) prio 255 cost 10 e26) pg 0 2019-12-07T19:34:03.613+0000 7fe2c7a66700 10 osd.0 26 handle_pg_query_nopg 1.8s5 2019-12-07T19:34:03.613+0000 7fe2c7a66700 10 osd.0 26 pg 1.8s5 dne 2019-12-07T19:34:03.613+0000 7fe2c7a66700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify((query:26 sent:26 1.8s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 5->0 ([0,0] all_participants= intervals=)) epoch 26) v7 -- 0x558e817b4960 con 0x558e818d7c00 2019-12-07T19:34:03.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:03.621+0000 7fe2e2ca2700 10 osd.0 26 tick_without_osd_lock 2019-12-07T19:34:03.621+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fd57000/0x40000000/0x1680000000, data 0x27c1c2/0x2a7000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:03.621+0000 7fe2e2ca2700 25 osd.0 26 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254252+0000 last_rx_front 2019-12-07T19:34:02.254310+0000 2019-12-07T19:34:03.621+0000 7fe2e2ca2700 25 osd.0 26 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254273+0000 last_rx_front 2019-12-07T19:34:02.254232+0000 2019-12-07T19:34:03.621+0000 7fe2e2ca2700 25 osd.0 26 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254209+0000 last_rx_front 2019-12-07T19:34:02.254194+0000 2019-12-07T19:34:03.621+0000 7fe2e2ca2700 25 osd.0 26 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254409+0000 last_rx_front 2019-12-07T19:34:02.254290+0000 2019-12-07T19:34:03.621+0000 7fe2e2ca2700 25 osd.0 26 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254354+0000 last_rx_front 2019-12-07T19:34:02.254301+0000 2019-12-07T19:34:03.621+0000 7fe2e2ca2700 25 osd.0 26 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254368+0000 last_rx_front 2019-12-07T19:34:02.254333+0000 2019-12-07T19:34:03.621+0000 7fe2e2ca2700 20 osd.0 26 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:03.621+0000 7fe2e2ca2700 10 osd.0 26 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:03.621+0000 7fe2e2ca2700 20 osd.0 26 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:03.621+0000 7fe2e2ca2700 10 osd.0 26 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:03.653+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.687+0000 7fe2e4535700 10 osd.0 26 tick 2019-12-07T19:34:03.687+0000 7fe2e4535700 10 osd.0 26 do_waiters -- start 2019-12-07T19:34:03.687+0000 7fe2e4535700 10 osd.0 26 do_waiters -- finish 2019-12-07T19:34:03.687+0000 7fe2e4535700 20 osd.0 26 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:03.703+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.753+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.803+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.853+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.903+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:03.942+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.5 172.21.15.109:0/36936 8 ==== osd_ping(ping e26 up_from 13 ping_stamp 2019-12-07T19:34:03.943243+0000/27.409567154s send_stamp 27.409567154s delta_ub 0.011839041s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83ca8c00 con 0x558e8182c800 2019-12-07T19:34:03.942+0000 7fe2e7d3c700 20 osd.0 26 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011361551s,0.011839041s]) 2019-12-07T19:34:03.942+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e26 up_from 13 ping_stamp 2019-12-07T19:34:03.943243+0000/27.409567154s send_stamp 27.398205603s delta_ub 18446744073.698190065s) v5 -- 0x558e83ca8400 con 0x558e8182c800 2019-12-07T19:34:03.942+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.5 172.21.15.109:0/36936 8 ==== osd_ping(ping e26 up_from 13 ping_stamp 2019-12-07T19:34:03.943243+0000/27.409567154s send_stamp 27.409567154s delta_ub 0.011839041s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83ca8c00 con 0x558e8181b800 2019-12-07T19:34:03.942+0000 7fe2e7d3c700 20 osd.0 26 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011317829s,0.011839041s]) 2019-12-07T19:34:03.942+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e26 up_from 13 ping_stamp 2019-12-07T19:34:03.943243+0000/27.409567154s send_stamp 27.398249325s delta_ub 18446744073.698233787s) v5 -- 0x558e84053e00 con 0x558e8181b800 2019-12-07T19:34:03.946+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:03.953+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.003+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.054+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.104+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.154+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.204+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.254+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.304+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.349+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:04.354+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.404+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.454+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.505+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.554+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.603+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 66 ==== pg_notify2(1.2s0 1.2s0 (query:27 sent:27 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e27/27) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e8404a000 con 0x558e818d7000 2019-12-07T19:34:04.603+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 3(5) notify: (query:27 sent:27 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:04.603+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 3(5) notify: (query:27 sent:27 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.603+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.603+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 3(5) notify: (query:27 sent:27 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.603+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27)> waiting <> waiting_peering {} 2019-12-07T19:34:04.603+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 3(5) notify: (query:27 sent:27 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.603+0000 7fe2c6a64700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 27, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 3(5) notify: (query:27 sent:27 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.603+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.604+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 117 ==== PGlog(1.8s5 log log((0'0,23'14], crt=23'14) pi ([18,25] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([18,23] acting 1(0),2(3),3(4),4(1),5(2),6(5))) pg_lease(ru 0s ub 44.061823309s int 16s) e27/26) v6 ==== 6590+0+0 (unknown 0 0 0) 0x558e819d0000 con 0x558e818d7c00 2019-12-07T19:34:04.604+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.8s5 epoch_sent: 27 epoch_requested: 26 MLogRec from 1(0) log log((0'0,23'14], crt=23'14) pi ([18,25] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([18,23] acting 1(0),2(3),3(4),4(1),5(2),6(5))) pg_lease(ru 0s ub 44.061823309s int 16s) +create_info 2019-12-07T19:34:04.604+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 1(0) log log((0'0,23'14], crt=23'14) pi ([18,25] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([18,23] acting 1(0),2(3),3(4),4(1),5(2),6(5))) pg_lease(ru 0s ub 44.061823309s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.604+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.604+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 1(0) log log((0'0,23'14], crt=23'14) pi ([18,25] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([18,23] acting 1(0),2(3),3(4),4(1),5(2),6(5))) pg_lease(ru 0s ub 44.061823309s int 16s) +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.604+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.604+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 1(0) log log((0'0,23'14], crt=23'14) pi ([18,25] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([18,23] acting 1(0),2(3),3(4),4(1),5(2),6(5))) pg_lease(ru 0s ub 44.061823309s int 16s) +create_info) prio 255 cost 10 e27) pg 0 2019-12-07T19:34:04.604+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 map 27 > 26 2019-12-07T19:34:04.604+0000 7fe2cba6e700 20 osd.0 op_wq _add_slot_waiter 1.8s5 peering, item epoch is 27, will wait on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 1(0) log log((0'0,23'14], crt=23'14) pi ([18,25] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([18,23] acting 1(0),2(3),3(4),4(1),5(2),6(5))) pg_lease(ru 0s ub 44.061823309s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.604+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.605+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 67 ==== osd_map(27..27 src has 1..27) v4 ==== 328+0+0 (unknown 0 0 0) 0x558e83f2a240 con 0x558e818d7000 2019-12-07T19:34:04.605+0000 7fe2d4a80700 20 osd.0 26 OSD::ms_dispatch: osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.605+0000 7fe2d4a80700 10 osd.0 26 do_waiters -- start 2019-12-07T19:34:04.605+0000 7fe2d4a80700 10 osd.0 26 do_waiters -- finish 2019-12-07T19:34:04.605+0000 7fe2d4a80700 20 osd.0 26 _dispatch 0x558e83f2a240 osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.605+0000 7fe2d4a80700 3 osd.0 26 handle_osd_map epochs [27,27], i have 26, src has [1,27] 2019-12-07T19:34:04.605+0000 7fe2d4a80700 10 osd.0 26 handle_osd_map got inc map for epoch 27 2019-12-07T19:34:04.605+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:35a0e826:::osdmap.26:0# 0x0~0 2019-12-07T19:34:04.605+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1671 size 0x1671 (5745) 2019-12-07T19:34:04.605+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:04.605+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f942a0 blob([0x108000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f94460 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:04.605+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f94540 blob([!~1000,0x109000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,671]) SharedBlob(0x558e83f94cb0 sbid 0x0)) need 0x1000~671 cache has 0x[] 2019-12-07T19:34:04.605+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f942a0 blob([0x108000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f94460 sbid 0x0)) need {<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:34:04.605+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~1000 2019-12-07T19:34:04.605+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f94540 blob([!~1000,0x109000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,671]) SharedBlob(0x558e83f94cb0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~671]} 2019-12-07T19:34:04.605+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.605+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.605+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 293316 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 68 ==== pg_info2(1.fs1 1.fs1 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404a000 con 0x558e818d7000 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.fs1 epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061994571s int 16s) 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061994571s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 69 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404a000 con 0x558e818d7000 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.1s1 epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061862442s int 16s) 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061862442s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 24 ==== osd_map(27..27 src has 1..27) v4 ==== 328+0+0 (crc 0 0 0) 0x558e83c66000 con 0x558e80f39000 2019-12-07T19:34:04.607+0000 7fe2d959f700 20 osd.0 26 OSD::ms_dispatch: osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.607+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061994571s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.607+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061994571s int 16s)) prio 255 cost 10 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.607+0000 7fe2c425f700 20 osd.0 op_wq _add_slot_waiter 1.fs1 peering, item epoch is 27, will wait on OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061994571s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.607+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061862442s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.607+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061862442s int 16s)) prio 255 cost 10 e27) pg 0x558e819e4000 2019-12-07T19:34:04.607+0000 7fe2c7265700 20 osd.0 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 27, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061862442s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 118 ==== pg_info2(1.9s3 1.9s3 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404a000 con 0x558e818d7c00 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.9s3 epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062707990s int 16s) 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062707990s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 119 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404a000 con 0x558e818d7c00 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.es4 epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063058723s int 16s) 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063058723s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063058723s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.607+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063058723s int 16s)) prio 255 cost 10 e27) pg 0x558e81a98000 2019-12-07T19:34:04.607+0000 7fe2c4a60700 20 osd.0 op_wq _add_slot_waiter 1.es4 peering, item epoch is 27, will wait on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063058723s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 120 ==== pg_info2(1.7s3 1.7s3 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404a000 con 0x558e818d7c00 2019-12-07T19:34:04.607+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 15 osd.0 26 enqueue_peering_evt 1.7s3 epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063255566s int 16s) 2019-12-07T19:34:04.607+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063255566s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063255566s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.607+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063255566s int 16s)) prio 255 cost 10 e27) pg 0x558e8122e000 2019-12-07T19:34:04.607+0000 7fe2c8267700 20 osd.0 op_wq _add_slot_waiter 1.7s3 peering, item epoch is 27, will wait on OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063255566s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.607+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f942a0 blob([0x108000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f94460 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x0:0~1000]} 2019-12-07T19:34:04.607+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e83f94478 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:04.607+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062707990s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.607+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.607+0000 7fe2d4a80700 20 _add level 1 near 0 on buffer(0x558e83d10990 space 0x558e83f94478 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.607+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062707990s int 16s)) prio 255 cost 10 e27) pg 0x558e8192b000 2019-12-07T19:34:04.607+0000 7fe2cb26d700 20 osd.0 op_wq _add_slot_waiter 1.9s3 peering, item epoch is 27, will wait on OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062707990s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.607+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.607+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f94540 blob([!~1000,0x109000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,671]) SharedBlob(0x558e83f94cb0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~671]} 2019-12-07T19:34:04.607+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e83f94cc8 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:04.607+0000 7fe2d4a80700 20 _add level 1 near 0 on buffer(0x558e83d11ef0 space 0x558e83f94cc8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.607+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:35a0e826:::osdmap.26:0# 0x0~1671 = 5745 2019-12-07T19:34:04.607+0000 7fe2d4a80700 10 osd.0 26 add_map_bl 26 5745 bytes 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 osd.0 26 got_full_map 27, nothing requested 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 osd.0 26 handle_osd_map pg_num_history pg_num_history(e27 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps purged_snaps {27={}} 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 osd.0 26 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e27 [1,27] lci=[13,27]) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e819a0600 seq 19 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c63b972f:::inc_osdmap.27:0# key 0x7f7fffffffffffffffc63b97'/!inc_osdmap.27!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c63b972f:::inc_osdmap.27:0# 0x0~11c 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1117 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c63b972f:::inc_osdmap.27:0# 0x0~11c - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~11c 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xee4 on front/back, now 0x0~1000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0600 1 blobs 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x10a000~1000/1000,1000,0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x10a000~1000] 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd2070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd2070 blob([0x10a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2150 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.blob(0x558e83dd2070) get_ref 0x0~11c Blob(0x558e83dd2070 blob([0x10a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2150 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.blob(0x558e83dd2070) get_ref init 0x1000, 1000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~11c: 0x0~11c Blob(0x558e83dd2070 blob([0x10a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x11c) SharedBlob(0x558e83dd2150 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e83dd2168 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x11c 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83b08108) dirty_range mark inline shard dirty 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c63b972f:::inc_osdmap.27:0# 0x0~11c = 0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3ea0e826:::osdmap.27:0# key 0x7f7fffffffffffffff3ea0e8'&!osdmap.27!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3ea0e826:::osdmap.27:0# 0x0~1671 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1118 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3ea0e826:::osdmap.27:0# 0x0~1671 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~671 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0x98f on front/back, now 0x0~1000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0600 2 blobs 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x10b000~2000/1000,2000,0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x10b000~2000] 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd2230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd20e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd2230 blob([0x10b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd20e0 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.blob(0x558e83dd2230) get_ref 0x0~1000 Blob(0x558e83dd2230 blob([0x10b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd20e0 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.blob(0x558e83dd2230) get_ref init 0x1000, 1000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e83dd2230 blob([0x10b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83dd20e0 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e83dd20f8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af9180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d0d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af9180 blob([!~1000,0x10c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d0d20 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.blob(0x558e83af9180) get_ref 0x1000~671 Blob(0x558e83af9180 blob([!~1000,0x10c000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e811d0d20 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.blob(0x558e83af9180) get_ref init 0x2000, 1000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~671: 0x1000~671 Blob(0x558e83af9180 blob([!~1000,0x10c000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,671]) SharedBlob(0x558e811d0d20 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e811d0d38 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1671 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191b788) dirty_range mark inline shard dirty 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3ea0e826:::osdmap.27:0# 0x0~1671 = 0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0600 0 blobs 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0600 0 blobs 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0600 cost 23510 (4 ios * 4000 + 7510 bytes) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0600 onodes 0x558e80251b80,0x558e817b9180,0x558e8191b680,0x558e83b08000 shared_blobs 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191b788) update #-1:3ea0e826:::osdmap.27:0# 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191b788) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3ea0e826:::osdmap.27:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83b08108) update #-1:c63b972f:::inc_osdmap.27:0# 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83b08108) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c63b972f:::inc_osdmap.27:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0600 allocated 0x[10a000~3000] released 0x[] 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 freelist allocate 0x10a000~3000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0600 prepare 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e819a0600 2019-12-07T19:34:04.608+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 121 ==== osd_map(27..27 src has 1..27) v4 ==== 328+0+0 (unknown 0 0 0) 0x558e83c66240 con 0x558e818d7c00 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 osd.0 26 OSD::ms_dispatch: osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 osd.0 26 do_waiters -- start 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 osd.0 26 do_waiters -- finish 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 osd.0 26 _dispatch 0x558e83c66240 osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.608+0000 7fe2d4a80700 3 osd.0 26 handle_osd_map epochs [27,27], i have 27, src has [1,27] 2019-12-07T19:34:04.608+0000 7fe2d4a80700 10 osd.0 26 no new maps here, dropping 2019-12-07T19:34:04.608+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0600 aio_wait 2019-12-07T19:34:04.608+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0600 2019-12-07T19:34:04.608+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0600 io_done 2019-12-07T19:34:04.608+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 67 ==== PGlog(1.3s5 log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4))) pg_lease(ru 0s ub 44.057306578s int 16s) e27/26) v6 ==== 5824+0+0 (unknown 0 0 0) 0x558e83e54800 con 0x558e818fe400 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.608+0000 7fe2e6d3a700 15 osd.0 26 enqueue_peering_evt 1.3s5 epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4))) pg_lease(ru 0s ub 44.057306578s int 16s) +create_info 2019-12-07T19:34:04.608+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4))) pg_lease(ru 0s ub 44.057306578s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.608+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8191b680 had 1 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b08000 had 1 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:04.608+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3860332544 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:04.608+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998661) bluestore 86 GiB free (0.959971), bluefs_ratio 0.0399501 2019-12-07T19:34:04.608+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:04.608+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:04.608+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.608+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 68 ==== PGlog(1.bs1 log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) e27/26) v6 ==== 7462+0+0 (unknown 0 0 0) 0x558e811b8000 con 0x558e818fe400 2019-12-07T19:34:04.608+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x79444~226c to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.608+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x79444 0x79444~226c to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x49444 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x444 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x6b0 and padding block with 0x950 2019-12-07T19:34:04.608+0000 7fe2e6d3a700 15 osd.0 26 enqueue_peering_evt 1.bs1 epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info 2019-12-07T19:34:04.608+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.608+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4))) pg_lease(ru 0s ub 44.057306578s int 16s) +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.608+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4))) pg_lease(ru 0s ub 44.057306578s int 16s) +create_info) prio 255 cost 10 e27) pg 0 2019-12-07T19:34:04.608+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 map 27 > 26 2019-12-07T19:34:04.608+0000 7fe2ca26b700 20 osd.0 op_wq _add_slot_waiter 1.3s5 peering, item epoch is 27, will wait on OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4))) pg_lease(ru 0s ub 44.057306578s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.608+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.608+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info) prio 255 cost 10 e27) pg 0 2019-12-07T19:34:04.608+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 map 27 > 26 2019-12-07T19:34:04.608+0000 7fe2ca26b700 20 osd.0 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 27, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.608+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.608+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 69 ==== pg_notify2(1.2s0 1.2s0 (query:27 sent:27 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e27/27) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e83eba700 con 0x558e818fe400 2019-12-07T19:34:04.608+0000 7fe2e6d3a700 15 osd.0 26 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 4(1) notify: (query:27 sent:27 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:04.608+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 4(1) notify: (query:27 sent:27 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.608+0000 7fe2d959f700 10 osd.0 26 do_waiters -- start 2019-12-07T19:34:04.608+0000 7fe2d959f700 10 osd.0 26 do_waiters -- finish 2019-12-07T19:34:04.608+0000 7fe2d959f700 20 osd.0 26 _dispatch 0x558e83c66000 osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.608+0000 7fe2d959f700 3 osd.0 26 handle_osd_map epochs [27,27], i have 27, src has [1,27] 2019-12-07T19:34:04.608+0000 7fe2d959f700 10 osd.0 26 no new maps here, dropping 2019-12-07T19:34:04.608+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 25 ==== PGlog(1.5s4 log log((0'0,23'7], crt=23'7) pi ([18,25] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1))) pg_lease(ru 0s ub 44.078908034s int 16s) e27/26) v6 ==== 3921+0+0 (unknown 0 0 0) 0x558e83e57000 con 0x558e818d6000 2019-12-07T19:34:04.608+0000 7fe2e753b700 15 osd.0 26 enqueue_peering_evt 1.5s4 epoch_sent: 27 epoch_requested: 26 MLogRec from 2(0) log log((0'0,23'7], crt=23'7) pi ([18,25] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1))) pg_lease(ru 0s ub 44.078908034s int 16s) +create_info 2019-12-07T19:34:04.608+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 2(0) log log((0'0,23'7], crt=23'7) pi ([18,25] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1))) pg_lease(ru 0s ub 44.078908034s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x7b6b0 2019-12-07T19:34:04.608+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.608+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.608+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000258661s (0.000004085s flush + 0.000254576s kv commit) 2019-12-07T19:34:04.608+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a0600> 2019-12-07T19:34:04.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.608+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0600 kv_submitted 2019-12-07T19:34:04.608+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0600 2019-12-07T19:34:04.608+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 2(0) log log((0'0,23'7], crt=23'7) pi ([18,25] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1))) pg_lease(ru 0s ub 44.078908034s int 16s) +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.608+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e819a0600 osr 0x558e8107a500 2019-12-07T19:34:04.608+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 2(0) log log((0'0,23'7], crt=23'7) pi ([18,25] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1))) pg_lease(ru 0s ub 44.078908034s int 16s) +create_info) prio 255 cost 10 e27) pg 0 2019-12-07T19:34:04.608+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 map 27 > 26 2019-12-07T19:34:04.608+0000 7fe2c5261700 20 osd.0 op_wq _add_slot_waiter 1.5s4 peering, item epoch is 27, will wait on OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 2(0) log log((0'0,23'7], crt=23'7) pi ([18,25] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1))) pg_lease(ru 0s ub 44.078908034s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.608+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) prepare_write seq 17 0x1b000~1000 crc 89c92970 2019-12-07T19:34:04.608+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.608+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) prepare_write seq 17 0x2000~1000 crc d757c9b3 2019-12-07T19:34:04.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.608+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {27=0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27)>} 2019-12-07T19:34:04.608+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 4(1) notify: (query:27 sent:27 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.608+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27)> waiting <> waiting_peering {27=0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27)>} 2019-12-07T19:34:04.608+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 4(1) notify: (query:27 sent:27 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.608+0000 7fe2caa6c700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 27, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 4(1) notify: (query:27 sent:27 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.608+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.608+0000 7fe2d7285700 10 osd.0 26 _committed_osd_maps 27..27 2019-12-07T19:34:04.608+0000 7fe2d7285700 10 osd.0 26 advance to epoch 27 (<= last 27 <= newest_map 27) 2019-12-07T19:34:04.608+0000 7fe2d7285700 7 osd.0 27 consume_map version 27 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:0.identify_splits_and_merges identify_splits_and_merges slot 1.8s5 has no pg and waiting_for_split 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0 27 identify_splits_and_merges 1.9s3 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0 27 identify_splits_and_merges 1.1s1 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0 27 identify_splits_and_merges 1.as3 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0 27 identify_splits_and_merges 1.2s0 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:3.identify_splits_and_merges identify_splits_and_merges slot 1.3s5 has no pg and waiting_for_split 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:3.identify_splits_and_merges identify_splits_and_merges slot 1.bs1 has no pg and waiting_for_split 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0 27 identify_splits_and_merges 1.bs3 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0 27 identify_splits_and_merges 1.4s1 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:5.identify_splits_and_merges identify_splits_and_merges slot 1.5s4 has no pg and waiting_for_split 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0 27 identify_splits_and_merges 1.ds5 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0 27 identify_splits_and_merges 1.es4 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0 27 identify_splits_and_merges 1.7s3 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0 27 identify_splits_and_merges 1.fs1 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0 27 prune_pg_created 2019-12-07T19:34:04.608+0000 7fe2d7285700 10 osd.0:0.consume_map 27 (was 26) 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 pending_peering first epoch 27 <= 27, requeueing 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {27=} 2019-12-07T19:34:04.608+0000 7fe2d7285700 10 osd.0:1.consume_map 27 (was 26) 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 pending_peering first epoch 27 <= 27, requeueing 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.9s3 to_process <> waiting <> waiting_peering {27=} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 pending_peering first epoch 27 <= 27, requeueing 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {27=} 2019-12-07T19:34:04.608+0000 7fe2d7285700 10 osd.0:2.consume_map 27 (was 26) 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 pending_peering first epoch 27 <= 27, requeueing 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.2s0 to_process <> waiting <> waiting_peering {27=0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 4(1) notify: (query:27 sent:27 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27)>} 2019-12-07T19:34:04.608+0000 7fe2d7285700 10 osd.0:3.consume_map 27 (was 26) 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 pending_peering first epoch 27 <= 27, requeueing 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.3s5 to_process <> waiting <> waiting_peering {27=} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 pending_peering first epoch 27 <= 27, requeueing 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {27=} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 2019-12-07T19:34:04.608+0000 7fe2d7285700 10 osd.0:4.consume_map 27 (was 26) 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:04.608+0000 7fe2d7285700 10 osd.0:5.consume_map 27 (was 26) 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 pending_peering first epoch 27 <= 27, requeueing 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:5._wake_pg_slot _wake_pg_slot 1.5s4 to_process <> waiting <> waiting_peering {27=} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:04.608+0000 7fe2d7285700 10 osd.0:6.consume_map 27 (was 26) 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 pending_peering first epoch 27 <= 27, requeueing 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {27=} 2019-12-07T19:34:04.608+0000 7fe2d7285700 10 osd.0:7.consume_map 27 (was 26) 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 pending_peering first epoch 27 <= 27, requeueing 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.7s3 to_process <> waiting <> waiting_peering {27=} 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 pending_peering first epoch 27 <= 27, requeueing 2019-12-07T19:34:04.608+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.fs1 to_process <> waiting <> waiting_peering {27=} 2019-12-07T19:34:04.608+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 70 ==== osd_map(27..27 src has 1..27) v4 ==== 328+0+0 (unknown 0 0 0) 0x558e8419f8c0 con 0x558e818fe400 2019-12-07T19:34:04.608+0000 7fe2d4a80700 20 osd.0 27 OSD::ms_dispatch: osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.608+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063058723s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.608+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 1(0) log log((0'0,23'14], crt=23'14) pi ([18,25] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([18,23] acting 1(0),2(3),3(4),4(1),5(2),6(5))) pg_lease(ru 0s ub 44.061823309s int 16s) +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.608+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063058723s int 16s)) prio 255 cost 10 e27) pg 0x558e81a98000 2019-12-07T19:34:04.608+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.608+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 1(0) log log((0'0,23'14], crt=23'14) pi ([18,25] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([18,23] acting 1(0),2(3),3(4),4(1),5(2),6(5))) pg_lease(ru 0s ub 44.061823309s int 16s) +create_info) prio 255 cost 10 e27) pg 0 2019-12-07T19:34:04.608+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 no pg, should create on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 1(0) log log((0'0,23'14], crt=23'14) pi ([18,25] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([18,23] acting 1(0),2(3),3(4),4(1),5(2),6(5))) pg_lease(ru 0s ub 44.061823309s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.608+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.608+0000 7fe2c8a68700 20 osd.0:6.update_pg_epoch 1.es4 26 -> 27 2019-12-07T19:34:04.608+0000 7fe2c8a68700 10 osd.0 pg_epoch: 26 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map [1,3,4,5,0,2]/[1,3,4,5,0,2147483647] -- 1/1 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 27 _make_pg 1.8s5 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 osd.0 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.609+0000 7fe2c7a66700 5 osd.0 pg_epoch: 26 pg[1.8(unlocked)] enter Initial 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063058723s int 16s) 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] state: got info from osd.1(0) 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 44.063058723s int 16s) 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] peer_clock_delta_lb 0.002061620s -> ruub 44.060997103s 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] proc_lease readable_until_ub now 44.060997103s 2019-12-07T19:34:04.609+0000 7fe2c8a68700 5 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.609788+0000 5 0.000100 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 osd.0 pg_epoch: 26 pg[1.8(unlocked)] enter NotTrimming 2019-12-07T19:34:04.609+0000 7fe2c8a68700 5 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach 1.8s5_head fresh osr 0x558e81967400 2019-12-07T19:34:04.609+0000 7fe2c8a68700 5 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped NOTIFY mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 26 pg[1.8s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role -1 up [1,4,5,2,3,0] acting [1,4,5,2,3,2147483647] history ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26 past_intervals ([18,25] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([18,23] acting 1(0),2(3),3(4),4(1),5(2),6(5))) 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped mbc={} ps=[2~1]] activate - no missing, moving last_complete 21'4 -> 21'4 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] init_hb_stamps now [] 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 28.064538419s + 0s) 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 osd.0 27 send_incremental_map 26 -> 27 to 0x558e818d7c00 v2:172.21.15.131:6823/37078 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:04.609+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c63b972f:::inc_osdmap.27:0# 0x0~0 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~11c size 0x11c (284) 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd2070 blob([0x10a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x11c) SharedBlob(0x558e83dd2150 sbid 0x0)) need 0x0~11c cache has 0x[0~11c] 2019-12-07T19:34:04.609+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c63b972f:::inc_osdmap.27:0# 0x0~11c = 284 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 osd.0 27 add_map_inc_bl 27 284 bytes 2019-12-07T19:34:04.609+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.8s5_head options 2019-12-07T19:34:04.609+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(27..27 src has 1..27) v4 -- 0x558e83d47b00 con 0x558e818d7c00 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 osd.0 27 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 26 -> 27 (shared) 2019-12-07T19:34:04.609+0000 7fe2c7a66700 5 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:04.609894+0000 0 0.000000 2019-12-07T19:34:04.609+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.es0 1.es0 (query:27 sent:27 1.es4( v 21'4 (0'0,21'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),6(5) intervals=([18,23] acting 0(4),1(0),3(1),4(2),5(3),6(5)))) e27/27) v1 -- 0x558e83c9f100 con 0x558e818d7c00 2019-12-07T19:34:04.609+0000 7fe2c7a66700 5 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] set_last_peering_reset 26 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e8417b800 seq 18 2019-12-07T19:34:04.609+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_activate_map: 26 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417b800 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417b800 onodes shared_blobs 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417b800 allocated 0x[] released 0x[] 2019-12-07T19:34:04.609+0000 7fe2c7a66700 5 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:04.609941+0000 1 0.000036 2019-12-07T19:34:04.609+0000 7fe2c7a66700 5 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 prepare 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417b800 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 io_done 2019-12-07T19:34:04.609+0000 7fe2c7a66700 5 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:34:04.609+0000 7fe2c7a66700 1 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.609+0000 7fe2c7a66700 5 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:34:04.609966+0000 0 0.000000 2019-12-07T19:34:04.609+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.609+0000 7fe2c7a66700 5 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061862442s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 26 while current is 26 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061862442s int 16s)) prio 255 cost 10 e27) pg 0x558e819e4000 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e8195e300 seq 1 2019-12-07T19:34:04.609+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.8s5_head bits 4 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.8s5_head bits 4 = 0 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.1s1 26 -> 27 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:10000000::::head# key 0x8580000000000000011000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 26 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map [3,0,2,4,5,1]/[3,0,2147483647,4,5,1] -- 3/3 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.609+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1119 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.609+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._infover' <- _infover 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.609+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.609+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._biginfo' <- _biginfo 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._epoch' <- _epoch 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061862442s int 16s) 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e300 cost 5934 (1 ios * 4000 + 1934 bytes) 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e300 onodes 0x558e8417c780 shared_blobs 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e8417c888) update 5#1:10000000::::head# 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e8417c888) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] state: got info from osd.3(0) 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:10000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e300 allocated 0x[] released 0x[] 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 44.061862442s int 16s) 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 prepare 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e300 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 io_done 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] peer_clock_delta_lb 0.002751230s -> ruub 44.059111212s 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] proc_lease readable_until_ub now 44.059111212s 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 27 handle_pg_create_info new pg pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] 2019-12-07T19:34:04.609+0000 7fe2c7265700 5 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.610100+0000 5 0.000094 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0:0._attach_pg 1.8s5 0x558e841a4000 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 osd.0:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 osd.0 27 identify_splits_and_merges 1.8s5 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0:0._prime_splits 2019-12-07T19:34:04.609+0000 7fe2c7265700 5 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.609+0000 7fe2c7265700 5 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped NOTIFY mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_advance_map: 27 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 osd.0:0.update_pg_epoch 1.8s5 26 -> 27 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped mbc={} ps=[2~1]] activate - no missing, moving last_complete 21'3 -> 21'3 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 26 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_advance_map [1,4,5,2,3,0]/[1,4,5,2,3,2147483647] -- 1/1 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 osd.0 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_activate_map: 27 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 27 send_incremental_map 26 -> 27 to 0x558e818d7000 v2:172.21.15.131:6812/37076 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:04.609+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(27..27 src has 1..27) v4 -- 0x558e8123a480 con 0x558e818d7000 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 osd.0 pg_epoch: 27 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 27 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 26 -> 27 (shared) 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 27 epoch_requested: 26 MLogRec from 1(0) log log((0'0,23'14], crt=23'14) pi ([18,25] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([18,23] acting 1(0),2(3),3(4),4(1),5(2),6(5))) pg_lease(ru 0s ub 44.061823309s int 16s) +create_info 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: got info+log from osd.1(0) 1.8s0( v 23'14 (0'0,23'14] local-lis/les=26/27 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) log((0'0,23'14], crt=23'14) 2019-12-07T19:34:04.609+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.1s0 1.1s0 (query:27 sent:27 1.1s1( v 21'3 (0'0,21'3] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),6(2) intervals=([18,23] acting 0(1),1(5),3(0),4(3),5(4),6(2)))) e27/27) v1 -- 0x558e83ebca00 con 0x558e818d7000 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 merge_log log((0'0,23'14], crt=23'14) from osd.1(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 merge_log extending head to 23'14 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ? 23'14 (21'3) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:181 0.000000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e8417a600 seq 16 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ? 21'13 (21'11) delete 1:14fc029d:::smithi13140796-41:head by client.4540.0:174 2019-12-07T19:33:58.823507+0000 0 ObjectCleanRegions clean_offsets: [688128~18446744073708863487], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ? 21'12 (21'10) modify 1:10f769a3:::smithi13140796-42:head by client.4540.0:125 2019-12-07T19:33:58.745945+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ? 21'11 (21'9) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:122 2019-12-07T19:33:58.741144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ? 21'10 (0'0) modify 1:10f769a3:::smithi13140796-42:head by client.4540.0:124 2019-12-07T19:33:58.745795+0000 0 ObjectCleanRegions clean_offsets: [704512~18446744073708847103], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ? 21'9 (0'0) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:121 2019-12-07T19:33:58.741112+0000 0 ObjectCleanRegions clean_offsets: [688128~18446744073708863487], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417a600 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417a600 onodes shared_blobs 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417a600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ? 21'8 (21'7) modify 1:10779340:::smithi13140796-32:head by client.4540.0:95 2019-12-07T19:33:58.703940+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ? 21'7 (0'0) modify 1:10779340:::smithi13140796-32:head by client.4540.0:94 2019-12-07T19:33:58.703907+0000 0 ObjectCleanRegions clean_offsets: [524288~18446744073709027327], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 prepare 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417a600 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ? 21'6 (21'5) modify 1:15219530:::smithi13140796-21:head by client.4540.0:62 2019-12-07T19:33:58.668552+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 io_done 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ? 21'5 (0'0) modify 1:15219530:::smithi13140796-21:head by client.4540.0:61 2019-12-07T19:33:58.668524+0000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ? 21'4 (21'2) modify 1:1353159e:::smithi13140796-11:head by client.4540.0:32 2019-12-07T19:33:58.635147+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062707990s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ? 21'3 (21'1) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:20 2019-12-07T19:33:58.628837+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062707990s int 16s)) prio 255 cost 10 e27) pg 0x558e8192b000 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ? 21'2 (0'0) modify 1:1353159e:::smithi13140796-11:head by client.4540.0:31 2019-12-07T19:33:58.635123+0000 0 ObjectCleanRegions clean_offsets: [180224~18446744073709371391], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 ? 21'1 (0'0) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:19 2019-12-07T19:33:58.628809+0000 0 ObjectCleanRegions clean_offsets: [114688~18446744073709436927], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 merge_log original_crt = 0'0 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 update missing, append 21'1 (0'0) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:19 2019-12-07T19:33:58.628809+0000 0 ObjectCleanRegions clean_offsets: [114688~18446744073709436927], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 update missing, append 21'2 (0'0) modify 1:1353159e:::smithi13140796-11:head by client.4540.0:31 2019-12-07T19:33:58.635123+0000 0 ObjectCleanRegions clean_offsets: [180224~18446744073709371391], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.9s3 26 -> 27 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 update missing, append 21'3 (21'1) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:20 2019-12-07T19:33:58.628837+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 26 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,2147483647] -- 1/1 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 update missing, append 21'4 (21'2) modify 1:1353159e:::smithi13140796-11:head by client.4540.0:32 2019-12-07T19:33:58.635147+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 update missing, append 21'5 (0'0) modify 1:15219530:::smithi13140796-21:head by client.4540.0:61 2019-12-07T19:33:58.668524+0000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 update missing, append 21'6 (21'5) modify 1:15219530:::smithi13140796-21:head by client.4540.0:62 2019-12-07T19:33:58.668552+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 update missing, append 21'7 (0'0) modify 1:10779340:::smithi13140796-32:head by client.4540.0:94 2019-12-07T19:33:58.703907+0000 0 ObjectCleanRegions clean_offsets: [524288~18446744073709027327], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 update missing, append 21'8 (21'7) modify 1:10779340:::smithi13140796-32:head by client.4540.0:95 2019-12-07T19:33:58.703940+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 3(5) notify: (query:27 sent:27 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 update missing, append 21'9 (0'0) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:121 2019-12-07T19:33:58.741112+0000 0 ObjectCleanRegions clean_offsets: [688128~18446744073708863487], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27)> waiting <> waiting_peering {} 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 3(5) notify: (query:27 sent:27 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 update missing, append 21'10 (0'0) modify 1:10f769a3:::smithi13140796-42:head by client.4540.0:124 2019-12-07T19:33:58.745795+0000 0 ObjectCleanRegions clean_offsets: [704512~18446744073708847103], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.062707990s int 16s) 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 update missing, append 21'11 (21'9) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:122 2019-12-07T19:33:58.741144+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] state: got info from osd.1(0) 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 update missing, append 21'12 (21'10) modify 1:10f769a3:::smithi13140796-42:head by client.4540.0:125 2019-12-07T19:33:58.745945+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 update missing, append 21'13 (21'11) delete 1:14fc029d:::smithi13140796-41:head by client.4540.0:174 2019-12-07T19:33:58.823507+0000 0 ObjectCleanRegions clean_offsets: [688128~18446744073708863487], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 44.062707990s int 16s) 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.2s0 26 -> 27 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 update missing, append 23'14 (21'3) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:181 0.000000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] peer_clock_delta_lb 0.002061620s -> ruub 44.060646370s 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3] -- 0/0 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 merge_log result log((0'0,23'14], crt=23'14) missing(6 may_include_deletes = 1) changed=1 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] proc_lease readable_until_ub now 44.060646370s 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: Peering advmap 2019-12-07T19:34:04.609+0000 7fe2c7265700 5 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.610366+0000 5 0.000090 2019-12-07T19:34:04.609+0000 7fe2c7a66700 5 osd.0 pg_epoch: 27 pg[1.8s5( v 23'14 lc 0'0 (0'0,23'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'14 unknown m=6 mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.610366+0000 3 0.000050 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] adjust_need_up_thru now 26, need_up_thru now false 2019-12-07T19:34:04.609+0000 7fe2c7265700 5 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.609+0000 7fe2c7a66700 5 osd.0 pg_epoch: 27 pg[1.8s5( v 23'14 lc 0'0 (0'0,23'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'14 unknown m=6 mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.609+0000 7fe2c7265700 5 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.609+0000 7fe2c7a66700 5 osd.0 pg_epoch: 27 pg[1.8s5( v 23'14 lc 0'0 (0'0,23'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'14 unknown m=6 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 23'14 lc 0'0 (0'0,23'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'14 unknown m=6 mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 osd.0 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped mbc={} ps=[2~1]] activate - no missing, moving last_complete 21'6 -> 21'6 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 23'14 lc 0'0 (0'0,23'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'14 unknown m=6 mbc={} ps=[2~1]] activate - not complete, missing(6 may_include_deletes = 1) 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 23'14 lc 21'3 (0'0,23'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'14 lcod 0'0 unknown m=6 mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:04.609+0000 7fe2c6a64700 7 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: handle ActMap primary 2019-12-07T19:34:04.609+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.609+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:27 sent:27 1.9s3( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 3->0 ([18,25] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([18,23] acting 0(3),1(0),4(1),5(2),6(5),7(4)))) e27/27) v1 -- 0x558e83c9e300 con 0x558e818d7c00 2019-12-07T19:34:04.609+0000 7fe2c6a64700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] publish_stats_to_osd 26:39 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e8417a900 seq 18 2019-12-07T19:34:04.609+0000 7fe2c6a64700 5 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] exit Started/Primary/Peering/WaitUpThru 2019-12-07T19:34:04.610449+0000 3 0.000272 2019-12-07T19:34:04.609+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] state: Leaving Peering 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417a900 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417a900 onodes shared_blobs 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417a900 allocated 0x[] released 0x[] 2019-12-07T19:34:04.609+0000 7fe2c6a64700 5 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped+peering mbc={} ps=[2~1]] exit Started/Primary/Peering 2019-12-07T19:34:04.610472+0000 0 0.000000 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a900 prepare 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417a900 2019-12-07T19:34:04.609+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a900 io_done 2019-12-07T19:34:04.609+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.609+0000 7fe2c6a64700 5 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] enter Started/Primary/Active 2019-12-07T19:34:04.609+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] state: In Active, about to call activate 2019-12-07T19:34:04.609+0000 7fe2d7285700 10 osd.0 27 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:04.609+0000 7fe2d7285700 15 osd.0 27 enqueue_peering_evt 1.9s3 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.609+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.609+0000 7fe2d7285700 15 osd.0 27 enqueue_peering_evt 1.1s1 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.609+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate - no missing, moving last_complete 23'10 -> 23'10 2019-12-07T19:34:04.609+0000 7fe2d7285700 15 osd.0 27 enqueue_peering_evt 1.as3 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.609+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.609+0000 7fe2d7285700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.609+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.609+0000 7fe2d7285700 15 osd.0 27 enqueue_peering_evt 1.bs3 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.609+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate - purged_snaps [2~1] removed_snaps [2~1] 2019-12-07T19:34:04.609+0000 7fe2d7285700 15 osd.0 27 enqueue_peering_evt 1.4s1 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.609+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.609+0000 7fe2d7285700 15 osd.0 27 enqueue_peering_evt 1.ds5 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:34:04.609+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.609+0000 7fe2d7285700 15 osd.0 27 enqueue_peering_evt 1.es4 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.609+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:04.609+0000 7fe2d7285700 15 osd.0 27 enqueue_peering_evt 1.7s3 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.609+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.609+0000 7fe2d7285700 15 osd.0 27 enqueue_peering_evt 1.fs1 epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.609+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:34:04.609+0000 7fe2d7285700 7 osd.0 27 activate_map version 27 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] recalc_readable_until readable_until[_ub] 0s (sent 44.065221680s) 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.1(2) 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558e81a98000 2019-12-07T19:34:04.609+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e8106c000 seq 2 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.1(2) is up to date, queueing in pending_activators 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.609+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkey_range 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.609+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:04.609+0000 7fe2c7a66700 20 bluestore.onode(0x558e8417c780).flush flush cnt:1 2019-12-07T19:34:04.609+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.609+0000 7fe2cc26f700 20 osd.0 27 agent_entry empty queue 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.1(2) 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) uptodate 2019-12-07T19:34:04.609+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.609+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061994571s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.3(5) 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.609+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.609+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061994571s int 16s)) prio 255 cost 10 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.3(5) is up to date, queueing in pending_activators 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.3(5) 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) uptodate 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.4(1) 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.609+0000 7fe2c425f700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.609+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.fs1 26 -> 27 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.4(1) is up to date, queueing in pending_activators 2019-12-07T19:34:04.609+0000 7fe2c425f700 10 osd.0 pg_epoch: 26 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map [3,0,5,4,1,2]/[3,0,5,4,1,2147483647] -- 3/3 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.4(1) 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) uptodate 2019-12-07T19:34:04.609+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.5(3) 1.2s3( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:04.609+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 copy_after v 0'0 2019-12-07T19:34:04.609+0000 7fe2c425f700 20 osd.0 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 copy_after copy log version 23'10 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 copy_after copy log version 21'9 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 copy_after copy log version 21'8 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 copy_after copy log version 21'7 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 copy_after copy log version 21'6 2019-12-07T19:34:04.609+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 copy_after copy log version 21'5 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 copy_after copy log version 21'4 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 copy_after copy log version 21'3 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 copy_after copy log version 21'2 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 copy_after copy log version 21'1 2019-12-07T19:34:04.609+0000 7fe2c6a64700 20 copy_up_to/copy_after earliest_dup_version 0 2019-12-07T19:34:04.609+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.609+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.609+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.5(3) sending log((0'0,23'10], crt=23'10) 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.061994571s int 16s) 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] state: got info from osd.3(0) 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.610+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- PGlog(1.2s3 log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) pg_lease(ru 0s ub 44.065221680s int 16s) e27/26) v6 -- 0x558e84040800 con 0x558e818d6800 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 44.061994571s int 16s) 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] peer_clock_delta_lb 0.002751230s -> ruub 44.059243341s 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] proc_lease readable_until_ub now 44.059243341s 2019-12-07T19:34:04.610+0000 7fe2c425f700 5 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.610732+0000 5 0.000074 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.5(3) 1.2s3( DNE v 23'10 lc 0'0 (0'0,23'10] local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) missing missing(4 may_include_deletes = 1) 2019-12-07T19:34:04.610+0000 7fe2c425f700 5 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.610+0000 7fe2c425f700 5 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.7(4) 1.2s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 copy_after v 0'0 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 copy_after copy log version 23'10 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 copy_after copy log version 21'9 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 copy_after copy log version 21'8 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 copy_after copy log version 21'7 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 copy_after copy log version 21'6 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped NOTIFY mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 copy_after copy log version 21'5 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 copy_after copy log version 21'4 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 copy_after copy log version 21'3 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 copy_after copy log version 21'2 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 copy_after copy log version 21'1 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 copy_up_to/copy_after earliest_dup_version 0 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped mbc={} ps=[2~1]] activate - no missing, moving last_complete 21'6 -> 21'6 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.7(4) sending log((0'0,23'10], crt=23'10) 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- PGlog(1.2s4 log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) pg_lease(ru 0s ub 44.065221680s int 16s) e27/26) v6 -- 0x558e84041000 con 0x558e818fe000 2019-12-07T19:34:04.610+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.fs0 1.fs0 (query:27 sent:27 1.fs1( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(1),1(4),2(5),3(0),4(3),5(2),6(5) intervals=([18,23] acting 0(1),1(4),3(0),4(3),5(2),6(5)))) e27/27) v1 -- 0x558e83ebd100 con 0x558e818d7000 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate peer osd.7(4) 1.2s4( DNE v 23'10 lc 0'0 (0'0,23'10] local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) missing missing(4 may_include_deletes = 1) 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e83d4d800 seq 22 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate setting up missing_loc from shard 0(0) 2019-12-07T19:34:04.610+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate setting up missing_loc from shard 1(2) 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate setting up missing_loc from shard 3(5) 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4d800 cost 5436 (1 ios * 4000 + 1436 bytes) 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4d800 onodes shared_blobs 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4d800 allocated 0x[] released 0x[] 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate setting up missing_loc from shard 4(1) 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 prepare 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4d800 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 io_done 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate setting up missing_loc from shard 5(3) 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063255566s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] activate setting up missing_loc from shard 7(4) 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063255566s int 16s)) prio 255 cost 10 e27) pg 0x558e8122e000 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] needs_recovery osd.5(3) has 4 missing 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={} ps=[2~1]] search_for_missing 1:41df217a:::smithi13140796-39:head 21'5 is on osd.0(0) 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.7s3 26 -> 27 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(0+0)=1},5={(0+0)=1}} ps=[2~1]] search_for_missing 1:41fe8a11:::smithi13140796-45:head 21'6 is on osd.0(0) 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 26 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2] -- 1/1 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=2},1={(0+0)=2},2={(0+0)=2},3={(0+0)=2},4={(0+0)=2},5={(0+0)=2}} ps=[2~1]] search_for_missing 1:4986b721:::smithi13140796-43:head 21'4 is on osd.0(0) 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(0+0)=3},3={(0+0)=3},4={(0+0)=3},5={(0+0)=3}} ps=[2~1]] add_source_info 1:4c9e444c:::smithi13140796-48:head delete, ignoring source 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.063255566s int 16s) 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] state: got info from osd.1(0) 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(0+0)=3},3={(0+0)=3},4={(0+0)=3},5={(0+0)=3}} ps=[2~1]] needs_recovery_map missing {1:41df217a:::smithi13140796-39:head=21'5 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:41fe8a11:::smithi13140796-45:head=21'6 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4986b721:::smithi13140796-43:head=21'4 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 44.063255566s int 16s) 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] peer_clock_delta_lb 0.002061620s -> ruub 44.061193946s 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] proc_lease readable_until_ub now 44.061193946s 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(0+0)=3},3={(0+0)=3},4={(0+0)=3},5={(0+0)=3}} ps=[2~1]] activate: adding 1(2) as a source 2019-12-07T19:34:04.610+0000 7fe2c425f700 5 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.610995+0000 5 0.000092 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(0+0)=3},3={(0+0)=3},4={(0+0)=3},5={(0+0)=3}} ps=[2~1]] search_for_missing 1:41df217a:::smithi13140796-39:head 21'5 is on osd.1(2) 2019-12-07T19:34:04.610+0000 7fe2c425f700 5 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.610+0000 7fe2c425f700 5 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped NOTIFY mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(0+0)=2,(1+0)=1},3={(0+0)=3},4={(0+0)=3},5={(0+0)=3}} ps=[2~1]] search_for_missing 1:41fe8a11:::smithi13140796-45:head 21'6 is on osd.1(2) 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped mbc={} ps=[2~1]] activate - no missing, moving last_complete 21'9 -> 21'9 2019-12-07T19:34:04.610+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(0+0)=1,(1+0)=2},3={(0+0)=3},4={(0+0)=3},5={(0+0)=3}} ps=[2~1]] search_for_missing 1:4986b721:::smithi13140796-43:head 21'4 is on osd.1(2) 2019-12-07T19:34:04.610+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.610+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558e8192b000 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(0+0)=3}} ps=[2~1]] add_source_info 1:4c9e444c:::smithi13140796-48:head delete, ignoring source 2019-12-07T19:34:04.610+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.610+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.610+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.610+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558e819e4000 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(0+0)=3}} ps=[2~1]] needs_recovery_map missing {1:41df217a:::smithi13140796-39:head=21'5 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:41fe8a11:::smithi13140796-45:head=21'6 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4986b721:::smithi13140796-43:head=21'4 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.610+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:27 sent:27 1.7s3( v 21'9 (0'0,21'9] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 3->0 ([18,25] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e27/27) v1 -- 0x558e80ff3800 con 0x558e818d7c00 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(0+0)=3}} ps=[2~1]] activate: adding 3(5) as a source 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e83d4db00 seq 27 2019-12-07T19:34:04.610+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:04.610+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(0+0)=3}} ps=[2~1]] search_for_missing 1:41df217a:::smithi13140796-39:head 21'5 is on osd.3(5) 2019-12-07T19:34:04.610+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:04.610+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(0+0)=2,(1+0)=1}} ps=[2~1]] search_for_missing 1:41fe8a11:::smithi13140796-45:head 21'6 is on osd.3(5) 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4db00 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4db00 onodes shared_blobs 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4db00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(0+0)=1,(1+0)=2}} ps=[2~1]] search_for_missing 1:4986b721:::smithi13140796-43:head 21'4 is on osd.3(5) 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4db00 prepare 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4db00 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4db00 io_done 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] add_source_info 1:4c9e444c:::smithi13140796-48:head delete, ignoring source 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] needs_recovery_map missing {1:41df217a:::smithi13140796-39:head=21'5 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:41fe8a11:::smithi13140796-45:head=21'6 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4986b721:::smithi13140796-43:head=21'4 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.610+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] activate: adding 4(1) as a source 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] search_for_missing 1:41df217a:::smithi13140796-39:head 21'5 is on osd.4(1) 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info) prio 255 cost 10 e27) pg 0 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=2,(1+0)=1},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] search_for_missing 1:41fe8a11:::smithi13140796-45:head 21'6 is on osd.4(1) 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 no pg, should create on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(0+0)=1,(1+0)=2},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] search_for_missing 1:4986b721:::smithi13140796-43:head 21'4 is on osd.4(1) 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] add_source_info 1:4c9e444c:::smithi13140796-48:head delete, ignoring source 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0 27 _make_pg 1.bs1 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] needs_recovery_map missing {1:41df217a:::smithi13140796-39:head=21'5 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:41fe8a11:::smithi13140796-45:head=21'6 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4986b721:::smithi13140796-43:head=21'4 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.610+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.b(unlocked)] enter Initial 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] activate: adding 5(3) as a source 2019-12-07T19:34:04.610+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558e8122e000 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] search_for_missing 1:41df217a:::smithi13140796-39:head 21'5 also missing on osd.5(3) 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:04.610+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] search_for_missing 1:41fe8a11:::smithi13140796-45:head 21'6 also missing on osd.5(3) 2019-12-07T19:34:04.610+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] search_for_missing 1:4986b721:::smithi13140796-43:head 21'4 also missing on osd.5(3) 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] add_source_info 1:4c9e444c:::smithi13140796-48:head delete, ignoring source 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.b(unlocked)] enter NotTrimming 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach 1.bs1_head fresh osr 0x558e83de5540 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] needs_recovery_map missing {1:41df217a:::smithi13140796-39:head=21'5 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:41fe8a11:::smithi13140796-45:head=21'6 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4986b721:::smithi13140796-43:head=21'4 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] activate: adding 7(4) as a source 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs1( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role -1 up [4,0,2,7,1,5] acting [4,2147483647,2,2147483647,1,5] history ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26 past_intervals ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 6 submitting 6 deferred done 0 stable 0 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] search_for_missing 1:41df217a:::smithi13140796-39:head 21'5 also missing on osd.7(4) 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1280 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] init_hb_stamps now [] 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] search_for_missing 1:41fe8a11:::smithi13140796-45:head 21'6 also missing on osd.7(4) 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 28.066031540s + 0s) 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2808 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] search_for_missing 1:4986b721:::smithi13140796-43:head 21'4 also missing on osd.7(4) 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] add_source_info 1:4c9e444c:::smithi13140796-48:head delete, ignoring source 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4088 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5368 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] needs_recovery_map missing {1:41df217a:::smithi13140796-39:head=21'5 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:41fe8a11:::smithi13140796-45:head=21'6 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4986b721:::smithi13140796-43:head=21'4 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.610+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs1_head options 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 6648 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] build_might_have_unfound 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 7939 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:04.611394+0000 0 0.000000 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:04.610+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 7962 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 7962 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7fe2c6a64700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] build_might_have_unfound: built 1(2),3(5),4(1),5(3),5(4),6(3),7(4) 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] set_last_peering_reset 26 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x7b6b0~1f1a to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x7b6b0 0x7b6b0~1f1a to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4b6b0 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x6b0 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x5ca and padding block with 0xa36 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] discover_all_missing 0 missing, 0 unfound 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_activate_map: 26 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] discover_all_missing: osd.1(2): we already have pg_missing_t 2019-12-07T19:34:04.610+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:04.611443+0000 1 0.000038 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] discover_all_missing: osd.3(5): we already have pg_missing_t 2019-12-07T19:34:04.610+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:34:04.610+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] discover_all_missing: osd.4(1): we already have pg_missing_t 2019-12-07T19:34:04.610+0000 7fe2c6263700 1 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:34:04.610+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:34:04.611475+0000 0 0.000000 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] discover_all_missing: osd.5(4): requesting pg_missing_t 2019-12-07T19:34:04.610+0000 7fe2c6263700 5 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 26 while current is 26 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e83d4cc00 seq 1 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] discover_all_missing skipping down osd.6(3) 2019-12-07T19:34:04.610+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.bs1_head bits 4 2019-12-07T19:34:04.610+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.bs1_head bits 4 = 0 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:04.610+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 31 ==== pg_notify2(1.2s0 1.2s0 (query:27 sent:27 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e27/27) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e8122b800 con 0x558e818d6800 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) get_onode oid 1#1:d0000000::::head# key 0x818000000000000001d000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) r -2 v.len 0 2019-12-07T19:34:04.610+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1120 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.610+0000 7fe2e753b700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 5(4) notify: (query:27 sent:27 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:04.610+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.610+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 5(4) notify: (query:27 sent:27 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._infover' <- _infover 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.610+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.610+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._biginfo' <- _biginfo 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._epoch' <- _epoch 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.610+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:04.610+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.610+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.610+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558e81938000 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4cc00 cost 5945 (1 ios * 4000 + 1945 bytes) 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4cc00 onodes 0x558e83c72780 shared_blobs 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83c72888) update 1#1:d0000000::::head# 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore.extentmap(0x558e83c72888) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:d0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4cc00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x7d5ca 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 prepare 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4cc00 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 io_done 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats no peer_missing found for 5(4) 2019-12-07T19:34:04.610+0000 7fe2c5a62700 10 osd.0 pg_epoch: 26 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.610+0000 7fe2c5a62700 20 osd.0:4.update_pg_epoch 1.4s1 26 -> 27 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 6 cleaned 0 in 0.000270585s (0.000000087s flush + 0.000270498s kv commit) 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.610+0000 7fe2c5a62700 10 osd.0 pg_epoch: 26 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0 27 handle_pg_create_info new pg pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1539 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0:3._attach_pg 1.bs1 0x558e83c78000 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c6263700 20 osd.0 27 identify_splits_and_merges 1.bs1 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0:3._prime_splits 2019-12-07T19:34:04.610+0000 7fe2c5a62700 10 osd.0 pg_epoch: 27 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.610+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:04.610+0000 7fe2c5a62700 10 osd.0 pg_epoch: 27 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.610+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4))) pg_lease(ru 0s ub 44.057306578s int 16s) +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1562 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1562 < min_flush_size 524288 2019-12-07T19:34:04.610+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.610+0000 7fe2c5a62700 20 osd.0 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.610+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4))) pg_lease(ru 0s ub 44.057306578s int 16s) +create_info) prio 255 cost 10 e27) pg 0 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1)=3} 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.610+0000 7fe2c5a62700 10 osd.0 pg_epoch: 27 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x7d5ca~61a to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.610+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_advance_map: 27 2019-12-07T19:34:04.610+0000 7fe2c5a62700 10 osd.0 pg_epoch: 27 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.610+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 no pg, should create on OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4))) pg_lease(ru 0s ub 44.057306578s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.610+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x7d5ca 0x7d5ca~61a to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~50000 x_off 0x4d5ca 2019-12-07T19:34:04.610+0000 7fe2c5a62700 20 osd.0 pg_epoch: 27 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 27 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x5ca 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.610+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xbe4 and padding block with 0x41c 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.610+0000 7fe2c5a62700 10 osd.0 pg_epoch: 27 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.610+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:04.610+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.610+0000 7fe2ca26b700 10 osd.0 27 _make_pg 1.3s5 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.610+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.610+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417b800,0x558e8195e300,0x558e8417a600,0x558e8417a900,0x558e83d4d800,0x558e83d4db00> 2019-12-07T19:34:04.610+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.610+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 kv_submitted 2019-12-07T19:34:04.610+0000 7fe2ca26b700 5 osd.0 pg_epoch: 26 pg[1.3(unlocked)] enter Initial 2019-12-07T19:34:04.610+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417b800 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 4 2019-12-07T19:34:04.610+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 finishing 2019-12-07T19:34:04.610+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417b800 onodes 2019-12-07T19:34:04.610+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b800 done 2019-12-07T19:34:04.610+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:04.610+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 4 2019-12-07T19:34:04.610+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417b800 [] 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.611+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 kv_submitted 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e300 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 finishing 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e300 onodes 0x558e8417c780 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 done 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:04.611+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 8 2019-12-07T19:34:04.611+0000 7fe2c8a68700 15 osd.0 27 enqueue_peering_evt 1.es4 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.611+0000 7fe2c8a68700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c000 prepare 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e300 [] 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.611+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 kv_submitted 2019-12-07T19:34:04.611+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417a600 2019-12-07T19:34:04.611+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 finishing 2019-12-07T19:34:04.611+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558e81a98000 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417a600 onodes 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a600 done 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417a600 [] 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a900 kv_submitted 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417a900 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a900 finishing 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417a900 onodes 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a900 done 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417a900 [] 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.611+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 kv_submitted 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4d800 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 osd.0 pg_epoch: 26 pg[1.3(unlocked)] enter NotTrimming 2019-12-07T19:34:04.611+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 finishing 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4d800 onodes 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4d800 done 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach 1.3s5_head fresh osr 0x558e83c6f2c0 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4d800 [] 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4db00 kv_submitted 2019-12-07T19:34:04.611+0000 7fe2c6a64700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 26:40 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4db00 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4db00 finishing 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4db00 onodes 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4db00 done 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4db00 [] 2019-12-07T19:34:04.611+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.611+0000 7fe2cb26d700 15 osd.0 27 enqueue_peering_evt 1.1s1 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.611+0000 7fe2cb26d700 15 osd.0 27 enqueue_peering_evt 1.9s3 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore.onode(0x558e8417c780).flush flush done 2019-12-07T19:34:04.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 26 pg[1.3s5( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role -1 up [4,1,2,5,7,0] acting [4,1,2,5,7,2147483647] history ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26 past_intervals ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4))) 2019-12-07T19:34:04.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 2(0) log log((0'0,23'7], crt=23'7) pi ([18,25] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1))) pg_lease(ru 0s ub 44.078908034s int 16s) +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558e819e4000 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 2(0) log log((0'0,23'7], crt=23'7) pi ([18,25] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1))) pg_lease(ru 0s ub 44.078908034s int 16s) +create_info) prio 255 cost 10 e27) pg 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:04.611+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 no pg, should create on OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 26 MLogRec from 2(0) log log((0'0,23'7], crt=23'7) pi ([18,25] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1))) pg_lease(ru 0s ub 44.078908034s int 16s) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.611+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'3 remapped mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] init_hb_stamps now [] 2019-12-07T19:34:04.611+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 28.066482609s + 0s) 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkey_range remove range start: 0x000000000000045f'.0000000000.00000000000000000000' end: 0x000000000000045f'.4294967295.18446744073709551615' 2019-12-07T19:34:04.611+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkey_range 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:04.611+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.1s0 1.1s0 1.1s1( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e81229c00 con 0x558e818d7000 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 27 _make_pg 1.5s4 2019-12-07T19:34:04.611+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:04.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7fe2ca26b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:04.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558e8192b000 2019-12-07T19:34:04.611+0000 7fe2c9269700 5 osd.0 pg_epoch: 26 pg[1.5(unlocked)] enter Initial 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:34:04.611+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.3s5_head options 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:34:04.611+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:34:04.611+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'6 remapped mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:34:04.611+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 ErasureCodeClay: parse (q,t,nu)=(2,3,0) 2019-12-07T19:34:04.611+0000 7fe2ca26b700 5 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:04.611835+0000 0 0.000000 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:34:04.611+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.9s0 1.9s0 1.9s3( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e81228000 con 0x558e818d7c00 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000021.00000000000000000010' <- 0000000021.00000000000000000010 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 ErasureCodePluginJerasure: factory: {k=4,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:04.611+0000 7fe2ca26b700 5 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000021.00000000000000000011' <- 0000000021.00000000000000000011 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000021.00000000000000000012' <- 0000000021.00000000000000000012 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] set_last_peering_reset 26 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000021.00000000000000000013' <- 0000000021.00000000000000000013 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000023.00000000000000000014' <- 0000000023.00000000000000000014 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._biginfo' <- _biginfo 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 ErasureCodePluginJerasure: factory: {k=2,m=2,plugin=jerasure,technique=reed_sol_van,w=8} 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._epoch' <- _epoch 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 ErasureCodeJerasure: technique=reed_sol_van 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.missing/0000000000000001.80EE9C20.head.smithi13140796-32..' <- missing/0000000000000001.80EE9C20.head.smithi13140796-32.. 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.missing/0000000000000001.80FE695C.head.smithi13140796-42..' <- missing/0000000000000001.80FE695C.head.smithi13140796-42.. 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.missing/0000000000000001.82F3049B.head.smithi13140796-41..' <- missing/0000000000000001.82F3049B.head.smithi13140796-41.. 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.missing/0000000000000001.8A489AC0.head.smithi13140796-21..' <- missing/0000000000000001.8A489AC0.head.smithi13140796-21.. 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0 pg_epoch: 26 pg[1.5(unlocked)] enter NotTrimming 2019-12-07T19:34:04.611+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.missing/0000000000000001.8C824871.head.smithi13140796-7..' <- missing/0000000000000001.8C824871.head.smithi13140796-7.. 2019-12-07T19:34:04.611+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 crt=21'4 remapped mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_attach 1.5s4_head fresh osr 0x558e811de3c0 2019-12-07T19:34:04.611+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.missing/0000000000000001.8CAC8A97.head.smithi13140796-11..' <- missing/0000000000000001.8CAC8A97.head.smithi13140796-11.. 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.611+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.611+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.es0 1.es0 1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e83c9c700 con 0x558e818d7c00 2019-12-07T19:34:04.611+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c000 cost 12557 (1 ios * 4000 + 8557 bytes) 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c000 onodes shared_blobs 2019-12-07T19:34:04.611+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c000 allocated 0x[] released 0x[] 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 26 pg[1.5s4( DNE empty local-lis/les=0/0 n=0 ec=0/0 lis/c=0/0 les/c/f=0/0/0 sis=0) []p? r=-1 lpr=0 crt=0'0 unknown mbc={}] init role -1 up [2,1,3,4,0,5] acting [2,2147483647,3,4,2147483647,5] history ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26 past_intervals ([18,25] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1))) 2019-12-07T19:34:04.611+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 prepare 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c000 2019-12-07T19:34:04.611+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 io_done 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] init_hb_stamps now [] 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] on_new_interval prior_readable_until_ub 0s (mnow 28.066610592s + 0s) 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] cancel_recovery 2019-12-07T19:34:04.611+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.611+0000 7fe2d4a80700 10 osd.0 27 do_waiters -- start 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] clear_recovery_state 2019-12-07T19:34:04.611+0000 7fe2d4a80700 10 osd.0 27 do_waiters -- finish 2019-12-07T19:34:04.611+0000 7fe2d4a80700 20 osd.0 27 _dispatch 0x558e8419f8c0 osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.611+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.611+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.611+0000 7fe2c9269700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:04.611+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.5s4_head options 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_activate_map: 26 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_initialize 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:04.611+0000 7fe2c9269700 5 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Initial 2019-12-07T19:34:04.611962+0000 0 0.000000 2019-12-07T19:34:04.611+0000 7fe2ca26b700 5 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:04.611967+0000 1 0.000116 2019-12-07T19:34:04.611+0000 7fe2c9269700 5 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Reset 2019-12-07T19:34:04.611+0000 7fe2ca26b700 5 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=0 pi=[18,26)/1 crt=0'0 unknown mbc={}] set_last_peering_reset 26 2019-12-07T19:34:04.611+0000 7fe2ca26b700 5 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] Clearing blocked outgoing recovery messages 2019-12-07T19:34:04.611+0000 7fe2ca26b700 1 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] Not blocking outgoing recovery messages 2019-12-07T19:34:04.611+0000 7fe2ca26b700 5 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:34:04.611992+0000 0 0.000000 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_activate_map: 26 2019-12-07T19:34:04.611+0000 7fe2ca26b700 5 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 26 while current is 26 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:04.611+0000 7fe2c9269700 5 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Reset 2019-12-07T19:34:04.612008+0000 1 0.000035 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e83c56f00 seq 1 2019-12-07T19:34:04.611+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.3s5_head bits 4 2019-12-07T19:34:04.611+0000 7fe2c9269700 5 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Started 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.3s5_head bits 4 = 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 5 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Start 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 1 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: transitioning to Stray 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) get_onode oid 5#1:c0000000::::head# key 0x858000000000000001c000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.611+0000 7fe2c9269700 5 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] exit Start 2019-12-07T19:34:04.612032+0000 0 0.000000 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) r -2 v.len 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 5 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] enter Started/Stray 2019-12-07T19:34:04.611+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1121 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 26 while current is 26 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:04.611+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._infover' <- _infover 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e8417b200 seq 1 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.5s4_head bits 4 2019-12-07T19:34:04.611+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._biginfo' <- _biginfo 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _create_collection 1.5s4_head bits 4 = 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._epoch' <- _epoch 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction collection hint objects is a no-op, pg_num 16 num_objects 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.5s4_head 0x558e8403fe00) get_onode oid 4#1:a0000000::::head# key 0x848000000000000001a000000021213dfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.5s4_head 0x558e8403fe00) r -2 v.len 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1122 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56f00 cost 5934 (1 ios * 4000 + 1934 bytes) 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56f00 onodes 0x558e8402ec80 shared_blobs 2019-12-07T19:34:04.611+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e8402ed88) update 5#1:c0000000::::head# 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e8402ed88) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._infover' <- _infover 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:c0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56f00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.611+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._biginfo' <- _biginfo 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 prepare 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56f00 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 io_done 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._epoch' <- _epoch 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.may_include_deletes_in_missing' <- may_include_deletes_in_missing 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 27 handle_pg_create_info new pg pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0:3._attach_pg 1.3s5 0x558e81231000 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417b200 cost 5945 (1 ios * 4000 + 1945 bytes) 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417b200 onodes 0x558e83cb3400 shared_blobs 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 osd.0 27 identify_splits_and_merges 1.3s5 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore.extentmap(0x558e83cb3508) update 4#1:a0000000::::head# 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0:3._prime_splits 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore.extentmap(0x558e83cb3508) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 4#1:a0000000::::head# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417b200 allocated 0x[] released 0x[] 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 prepare 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417b200 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 io_done 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_advance_map: 27 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.3s5 26 -> 27 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 26 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,2147483647] -- 4/4 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 osd.0 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 27 handle_pg_create_info new pg pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0:5._attach_pg 1.5s4 0x558e83d07000 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0:5._wake_pg_slot _wake_pg_slot 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_activate_map: 27 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0 27 identify_splits_and_merges 1.5s4 e26 to e27 pg_nums {18=16} 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0:5._prime_splits 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 osd.0 pg_epoch: 27 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,23'10], crt=23'10) pi ([18,25] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([18,23] acting 1(1),2(2),4(0),5(3),6(5),7(4))) pg_lease(ru 0s ub 44.057306578s int 16s) +create_info 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_advance_map: 27 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.5s4 26 -> 27 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 26 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_advance_map [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5] -- 2/2 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: got info+log from osd.4(0) 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) log((0'0,23'10], crt=23'10) 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 merge_log log((0'0,23'10], crt=23'10) from osd.4(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 merge_log extending head to 23'10 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 ? 23'10 (23'9) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:179 2019-12-07T19:34:00.651608+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 ? 23'9 (23'8) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:178 2019-12-07T19:34:00.651599+0000 0 ObjectCleanRegions clean_offsets: [0~999424,1376256~18446744073708175359], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 ? 23'8 (21'5) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:177 2019-12-07T19:34:00.649423+0000 0 ObjectCleanRegions clean_offsets: [0~442368,999424~18446744073708552191], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_activate_map: 27 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 ? 21'7 (21'2) delete 1:c87708e9:::smithi13140796-17:head by client.4540.0:154 2019-12-07T19:33:58.792683+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 ? 21'6 (21'4) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:86 2019-12-07T19:33:58.693762+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 osd.0 pg_epoch: 27 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 ? 21'5 (21'3) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:80 2019-12-07T19:33:58.687362+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 ? 21'4 (0'0) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:85 2019-12-07T19:33:58.693730+0000 0 ObjectCleanRegions clean_offsets: [475136~18446744073709076479], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 27 epoch_requested: 26 MLogRec from 2(0) log log((0'0,23'7], crt=23'7) pi ([18,25] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1))) pg_lease(ru 0s ub 44.078908034s int 16s) +create_info 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 ? 21'3 (0'0) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:79 2019-12-07T19:33:58.687333+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 ? 21'2 (21'1) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:50 2019-12-07T19:33:58.656793+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: got info+log from osd.2(0) 1.5s0( v 23'7 (0'0,23'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) log((0'0,23'7], crt=23'7) 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 ? 21'1 (0'0) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:49 2019-12-07T19:33:58.656740+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 merge_log original_crt = 0'0 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 merge_log log((0'0,23'7], crt=23'7) from osd.2(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 merge_log extending head to 23'7 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 update missing, append 21'1 (0'0) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:49 2019-12-07T19:33:58.656740+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 ? 23'7 (21'6) delete 1:a7686634:::smithi13140796-35:head by client.4540.0:185 2019-12-07T19:34:00.666376+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 ? 21'6 (21'5) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:104 2019-12-07T19:33:58.715053+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 update missing, append 21'2 (21'1) modify 1:c87708e9:::smithi13140796-17:head by client.4540.0:50 2019-12-07T19:33:58.656793+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 ? 21'5 (0'0) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:103 2019-12-07T19:33:58.715018+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 update missing, append 21'3 (0'0) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:79 2019-12-07T19:33:58.687333+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 ? 21'4 (21'3) modify 1:a8debf31:::smithi13140796-24:head by client.4540.0:71 2019-12-07T19:33:58.678863+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 ? 21'3 (0'0) modify 1:a8debf31:::smithi13140796-24:head by client.4540.0:70 2019-12-07T19:33:58.678831+0000 0 ObjectCleanRegions clean_offsets: [393216~18446744073709158399], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 update missing, append 21'4 (0'0) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:85 2019-12-07T19:33:58.693730+0000 0 ObjectCleanRegions clean_offsets: [475136~18446744073709076479], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 ? 21'2 (21'1) modify 1:a9dea763:::smithi13140796-12:head by client.4540.0:35 2019-12-07T19:33:58.636666+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 ? 21'1 (0'0) modify 1:a9dea763:::smithi13140796-12:head by client.4540.0:34 2019-12-07T19:33:58.636620+0000 0 ObjectCleanRegions clean_offsets: [196608~18446744073709355007], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 update missing, append 21'5 (21'3) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:80 2019-12-07T19:33:58.687362+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 merge_log original_crt = 0'0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 update missing, append 21'6 (21'4) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:86 2019-12-07T19:33:58.693762+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 update missing, append 21'1 (0'0) modify 1:a9dea763:::smithi13140796-12:head by client.4540.0:34 2019-12-07T19:33:58.636620+0000 0 ObjectCleanRegions clean_offsets: [196608~18446744073709355007], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 update missing, append 21'7 (21'2) delete 1:c87708e9:::smithi13140796-17:head by client.4540.0:154 2019-12-07T19:33:58.792683+0000 0 ObjectCleanRegions clean_offsets: [278528~18446744073709273087], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 update missing, append 21'2 (21'1) modify 1:a9dea763:::smithi13140796-12:head by client.4540.0:35 2019-12-07T19:33:58.636666+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 update missing, append 23'8 (21'5) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:177 2019-12-07T19:34:00.649423+0000 0 ObjectCleanRegions clean_offsets: [0~442368,999424~18446744073708552191], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 update missing, append 23'9 (23'8) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:178 2019-12-07T19:34:00.651599+0000 0 ObjectCleanRegions clean_offsets: [0~999424,1376256~18446744073708175359], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 update missing, append 21'3 (0'0) modify 1:a8debf31:::smithi13140796-24:head by client.4540.0:70 2019-12-07T19:33:58.678831+0000 0 ObjectCleanRegions clean_offsets: [393216~18446744073709158399], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 update missing, append 23'10 (23'9) modify 1:cd22b62d:::smithi13140796-27:head by client.4540.0:179 2019-12-07T19:34:00.651608+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 update missing, append 21'4 (21'3) modify 1:a8debf31:::smithi13140796-24:head by client.4540.0:71 2019-12-07T19:33:58.678863+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 merge_log result log((0'0,23'10], crt=23'10) missing(3 may_include_deletes = 1) changed=1 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 update missing, append 21'5 (0'0) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:103 2019-12-07T19:33:58.715018+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 update missing, append 21'6 (21'5) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:104 2019-12-07T19:33:58.715053+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 5 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 0'0 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 unknown m=3 mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.612323+0000 3 0.000050 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 update missing, append 23'7 (21'6) delete 1:a7686634:::smithi13140796-35:head by client.4540.0:185 2019-12-07T19:34:00.666376+0000 0 ObjectCleanRegions clean_offsets: [573440~18446744073708978175], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.611+0000 7fe2ca26b700 5 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 0'0 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 unknown m=3 mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 merge_log result log((0'0,23'7], crt=23'7) missing(3 may_include_deletes = 1) changed=1 2019-12-07T19:34:04.611+0000 7fe2ca26b700 5 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 0'0 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 unknown m=3 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.611+0000 7fe2c9269700 5 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 0'0 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'7 unknown m=3 mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.612343+0000 3 0.000050 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 0'0 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 unknown m=3 mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.611+0000 7fe2c9269700 5 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 0'0 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'7 unknown m=3 mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 0'0 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 unknown m=3 mbc={} ps=[2~1]] activate - not complete, missing(3 may_include_deletes = 1) 2019-12-07T19:34:04.611+0000 7fe2c9269700 5 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 0'0 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'7 unknown m=3 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 lcod 0'0 unknown m=3 mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 0'0 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'7 unknown m=3 mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.611+0000 7fe2ca26b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 0'0 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'7 unknown m=3 mbc={} ps=[2~1]] activate - not complete, missing(3 may_include_deletes = 1) 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'7 lcod 0'0 unknown m=3 mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.611+0000 7fe2c9269700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.611+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.611+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.611+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558e819e7000 2019-12-07T19:34:04.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e83c56600 seq 2 2019-12-07T19:34:04.611+0000 7fe2c5261700 10 osd.0 pg_epoch: 26 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.611+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkey_range 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:04.611+0000 7fe2c5261700 20 osd.0:5.update_pg_epoch 1.ds5 26 -> 27 2019-12-07T19:34:04.611+0000 7fe2ca26b700 20 bluestore.onode(0x558e8402ec80).flush flush cnt:1 2019-12-07T19:34:04.611+0000 7fe2c5261700 10 osd.0 pg_epoch: 26 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:04.611+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.611+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e819a1b00 seq 2 2019-12-07T19:34:04.611+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.611+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkey_range 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.611+0000 7fe2c5261700 20 osd.0 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.611+0000 7fe2c9269700 20 bluestore.onode(0x558e83cb3400).flush flush cnt:1 2019-12-07T19:34:04.611+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.611+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.611+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.611+0000 7fe2c5261700 20 osd.0 pg_epoch: 27 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 27 2019-12-07T19:34:04.611+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:04.611+0000 7fe2c6a64700 5 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] enter Started/Primary/Active/Activating 2019-12-07T19:34:04.611+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.611+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:04.611+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.611+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.611+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x7dbe4 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.611+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.611+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.611+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000913694s (0.000000073s flush + 0.000913621s kv commit) 2019-12-07T19:34:04.611+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:04.611+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 3(5) notify: (query:27 sent:27 1.2s5( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 5->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8405 < min_flush_size 524288 2019-12-07T19:34:04.611+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] state: Active: got notify from 3(5), already have info from that osd, ignoring 2019-12-07T19:34:04.611+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.611+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 -- 0x558e83c46e00 con 0x558e818d7c00 2019-12-07T19:34:04.611+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 9244 < min_flush_size 524288 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 9944 < min_flush_size 524288 2019-12-07T19:34:04.611+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:04.611+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 -- 0x558e8404a700 con 0x558e818d7000 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 11483 < min_flush_size 524288 2019-12-07T19:34:04.611+0000 7fe2c6a64700 10 osd.0 27 send_incremental_map 26 -> 27 to 0x558e818fe400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:04.611+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 11506 < min_flush_size 524288 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 11506 < min_flush_size 524288 2019-12-07T19:34:04.611+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(27..27 src has 1..27) v4 -- 0x558e83dd1b00 con 0x558e818fe400 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x7dbe4~2cf2 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.611+0000 7fe2c6a64700 10 osd.0 27 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 26 -> 27 (shared) 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x7dbe4 0x7dbe4~2cf2 to file(ino 16 size 0x80000 mtime 2019-12-07T19:34:02.646549+0000 allocated 80000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~50000]) 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs _allocate len 0x8d6 from 0 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs _allocate len 0x8d6 from 1 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd1b0000 2019-12-07T19:34:04.611+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 -- 0x558e8404ca00 con 0x558e818fe400 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd1b0000~10000/10000,10000,acd1b0000 2019-12-07T19:34:04.611+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x90000 to include allocated 2019-12-07T19:34:04.611+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 30 (was clean) 2019-12-07T19:34:04.611+0000 7fe2c6a64700 10 osd.0 27 send_incremental_map 26 -> 27 to 0x558e818d6800 v2:172.21.15.109:6802/36936 2019-12-07T19:34:04.611+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.611+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x4dbe4 2019-12-07T19:34:04.611+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xbe4 2019-12-07T19:34:04.611+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.611+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x8d6 and padding block with 0x72a 2019-12-07T19:34:04.611+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(27..27 src has 1..27) v4 -- 0x558e819b2fc0 con 0x558e818d6800 2019-12-07T19:34:04.611+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x90000 2019-12-07T19:34:04.611+0000 7fe2c6a64700 10 osd.0 27 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 26 -> 27 (shared) 2019-12-07T19:34:04.611+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.2s4 1.2s4 query(fulllog 0'0 epoch_sent 27) e27/27) v1 -- 0x558e8417d900 con 0x558e818d6800 2019-12-07T19:34:04.611+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:04.611+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e811fb500 seq 25 2019-12-07T19:34:04.611+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:04.611+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:04.611+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:04.611+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:04.611+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:04.611+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fb500 cost 5574 (1 ios * 4000 + 1574 bytes) 2019-12-07T19:34:04.611+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fb500 onodes shared_blobs 2019-12-07T19:34:04.611+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fb500 allocated 0x[] released 0x[] 2019-12-07T19:34:04.611+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 prepare 2019-12-07T19:34:04.611+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fb500 2019-12-07T19:34:04.611+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 io_done 2019-12-07T19:34:04.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 4(1) notify: (query:27 sent:27 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27)> waiting <> waiting_peering {} 2019-12-07T19:34:04.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 4(1) notify: (query:27 sent:27 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 4(1) notify: (query:27 sent:27 1.2s1( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 1->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:04.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] state: Active: got notify from 4(1), already have info from that osd, ignoring 2019-12-07T19:34:04.612+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4cc00> 2019-12-07T19:34:04.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 kv_submitted 2019-12-07T19:34:04.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4cc00 2019-12-07T19:34:04.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 finishing 2019-12-07T19:34:04.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4cc00 onodes 0x558e83c72780 2019-12-07T19:34:04.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558e8193b000 2019-12-07T19:34:04.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4cc00 done 2019-12-07T19:34:04.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:04.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4cc00 [] 2019-12-07T19:34:04.612+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore.onode(0x558e8402ec80).flush flush done 2019-12-07T19:34:04.612+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.612+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.612+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.612+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.612+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.612+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.612+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkey_range remove range start: 0x0000000000000461'.0000000000.00000000000000000000' end: 0x0000000000000461'.4294967295.18446744073709551615' 2019-12-07T19:34:04.612+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 5(4) notify: (query:27 sent:27 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.612+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27)> waiting <> waiting_peering {} 2019-12-07T19:34:04.612+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkey_range 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:04.612+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 5(4) notify: (query:27 sent:27 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.612+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:04.612+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 5(4) notify: (query:27 sent:27 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 4->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:34:04.612+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] state: Active: got notify from 5(4), already have info from that osd, ignoring 2019-12-07T19:34:04.612+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:34:04.612+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.0000000023.00000000000000000008' <- 0000000023.00000000000000000008 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.0000000023.00000000000000000009' <- 0000000023.00000000000000000009 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.0000000023.00000000000000000010' <- 0000000023.00000000000000000010 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._biginfo' <- _biginfo 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._epoch' <- _epoch 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.missing/0000000000000001.3137CAAC.head.smithi13140796-29..' <- missing/0000000000000001.3137CAAC.head.smithi13140796-29.. 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.missing/0000000000000001.31EE0179.head.smithi13140796-17..' <- missing/0000000000000001.31EE0179.head.smithi13140796-17.. 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.missing/0000000000000001.3B44D64B.head.smithi13140796-27..' <- missing/0000000000000001.3B44D64B.head.smithi13140796-27.. 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.612+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:04.612+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56600 cost 11128 (1 ios * 4000 + 7128 bytes) 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56600 onodes shared_blobs 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.612+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 prepare 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56600 2019-12-07T19:34:04.612+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 io_done 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 NullEvt) prio 255 cost 10 e27) pg 0x558e81928000 2019-12-07T19:34:04.612+0000 7fe2ca26b700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map: 27 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs3 26 -> 27 2019-12-07T19:34:04.612+0000 7fe2ca26b700 10 osd.0 pg_epoch: 26 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5] -- 4/4 2019-12-07T19:34:04.612+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.612+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.as3 26 -> 27 2019-12-07T19:34:04.612+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 osd.0 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.612+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 26 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] handle_advance_map [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647] -- 1/1 2019-12-07T19:34:04.612+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:04.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.612+0000 7fe2ca26b700 20 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.612+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:04.612+0000 7fe2c6a64700 20 osd.0 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.612+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.612+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:04.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] handle_activate_map: 27 2019-12-07T19:34:04.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:04.612+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:27 sent:27 1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 3->0 ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e27/27) v1 -- 0x558e83ebd800 con 0x558e818fe400 2019-12-07T19:34:04.612+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.612+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 NullEvt 2019-12-07T19:34:04.612+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.612+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.as0 1.as0 (query:27 sent:27 1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 3->0 ([18,25] all_participants=0(3),1(0),2(2),3(5),4(1),4(5),5(4),6(1) intervals=([18,23] acting 0(3),1(0),2(2),4(5),5(4),6(1)))) e27/27) v1 -- 0x558e8404bc00 con 0x558e818d7c00 2019-12-07T19:34:04.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 osd.0:3.update_pg_epoch 1.bs1 26 -> 27 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 26 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_advance_map [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5] -- 4/4 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: Started advmap 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] check_recovery_sources no source osds () went down 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 osd.0 27 advance_pg new pool opts old pool opts 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] handle_activate_map: 27 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 osd.0 pg_epoch: 27 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] activate_map: Not dirtying info: last_persisted is 26 while current is 27 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] do_peering_event: epoch_sent: 27 epoch_requested: 26 MLogRec from 4(0) log log((0'0,21'12], crt=21'12) pi ([18,25] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) pg_lease(ru 0s ub 44.057430495s int 16s) +create_info 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( empty local-lis/les=0/0 n=0 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=0'0 unknown mbc={}] state: got info+log from osd.4(0) 1.bs0( v 21'12 (0'0,21'12] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) log((0'0,21'12], crt=21'12) 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 merge_log log((0'0,21'12], crt=21'12) from osd.4(0) into log((0'0,0'0], crt=0'0) 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 merge_log extending head to 21'12 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 ? 21'12 (21'11) delete 1:d07269f7:::smithi13140796-28:head by client.4540.0:165 2019-12-07T19:33:58.821029+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 ? 21'11 (21'4) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:158 2019-12-07T19:33:58.802344+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 ? 21'10 (21'7) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:148 2019-12-07T19:33:58.786376+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 ? 21'9 (21'8) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:152 2019-12-07T19:33:58.792599+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 ? 21'8 (21'6) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:151 2019-12-07T19:33:58.792585+0000 0 ObjectCleanRegions clean_offsets: [0~458752,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 ? 21'7 (21'5) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:147 2019-12-07T19:33:58.786362+0000 0 ObjectCleanRegions clean_offsets: [0~442368,819200~18446744073708732415], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 ? 21'6 (0'0) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:150 2019-12-07T19:33:58.789897+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 ? 21'5 (0'0) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:146 2019-12-07T19:33:58.783523+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 ? 21'4 (21'3) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:83 2019-12-07T19:33:58.690784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 ? 21'3 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:82 2019-12-07T19:33:58.690760+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 ? 21'2 (21'1) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:53 2019-12-07T19:33:58.659098+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 ? 21'1 (0'0) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:52 2019-12-07T19:33:58.659063+0000 0 ObjectCleanRegions clean_offsets: [294912~18446744073709256703], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 merge_log cut point (usually last shared) is 0'0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 merge_log original_crt = 0'0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 update missing, append 21'1 (0'0) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:52 2019-12-07T19:33:58.659063+0000 0 ObjectCleanRegions clean_offsets: [294912~18446744073709256703], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 update missing, append 21'2 (21'1) modify 1:db85c185:::smithi13140796-18:head by client.4540.0:53 2019-12-07T19:33:58.659098+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 update missing, append 21'3 (0'0) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:82 2019-12-07T19:33:58.690760+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 update missing, append 21'4 (21'3) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:83 2019-12-07T19:33:58.690784+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 update missing, append 21'5 (0'0) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:146 2019-12-07T19:33:58.783523+0000 0 ObjectCleanRegions clean_offsets: [442368~18446744073709109247], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 update missing, append 21'6 (0'0) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:150 2019-12-07T19:33:58.789897+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 update missing, append 21'7 (21'5) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:147 2019-12-07T19:33:58.786362+0000 0 ObjectCleanRegions clean_offsets: [0~442368,819200~18446744073708732415], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 update missing, append 21'8 (21'6) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:151 2019-12-07T19:33:58.792585+0000 0 ObjectCleanRegions clean_offsets: [0~458752,835584~18446744073708716031], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 update missing, append 21'9 (21'8) modify 1:dce63cc3:::smithi13140796-50:head by client.4540.0:152 2019-12-07T19:33:58.792599+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 update missing, append 21'10 (21'7) modify 1:d313ddd1:::smithi13140796-49:head by client.4540.0:148 2019-12-07T19:33:58.786376+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 update missing, append 21'11 (21'4) modify 1:d07269f7:::smithi13140796-28:head by client.4540.0:158 2019-12-07T19:33:58.802344+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 update missing, append 21'12 (21'11) delete 1:d07269f7:::smithi13140796-28:head by client.4540.0:165 2019-12-07T19:33:58.821029+0000 0 ObjectCleanRegions clean_offsets: [458752~18446744073709092863], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 merge_log result log((0'0,21'12], crt=21'12) missing(4 may_include_deletes = 1) changed=1 2019-12-07T19:34:04.613+0000 7fe2c6263700 5 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 unknown m=4 mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.613995+0000 3 0.000061 2019-12-07T19:34:04.613+0000 7fe2c8267700 15 osd.0 27 enqueue_peering_evt 1.fs1 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.613+0000 7fe2c6263700 5 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 unknown m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.613+0000 7fe2c8267700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.613+0000 7fe2c8267700 15 osd.0 27 enqueue_peering_evt 1.7s3 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.613+0000 7fe2c6263700 5 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 unknown m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.613+0000 7fe2c8267700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.613+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.613+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 unknown m=4 mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.613+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.613+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 0'0 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 unknown m=4 mbc={} ps=[2~1]] activate - not complete, missing(4 may_include_deletes = 1) 2019-12-07T19:34:04.613+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 lcod 0'0 unknown m=4 mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.613+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 crt=21'6 remapped mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.613+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 0'0, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.613+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:04.613+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.fs0 1.fs0 1.fs1( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e81a58000 con 0x558e818d7000 2019-12-07T19:34:04.613+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.613+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.613+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.613+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558e8122e000 2019-12-07T19:34:04.613+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.613+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'9 remapped mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.613+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:04.613+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.7s0 1.7s0 1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e81a58700 con 0x558e818d7c00 2019-12-07T19:34:04.613+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e83d4d500 seq 2 2019-12-07T19:34:04.613+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkey_range 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore.onode(0x558e83c72780).flush flush done 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkey_range remove range start: 0x0000000000000460'.0000000000.00000000000000000000' end: 0x0000000000000460'.4294967295.18446744073709551615' 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkey_range 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.0000000021.00000000000000000007' <- 0000000021.00000000000000000007 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.0000000021.00000000000000000008' <- 0000000021.00000000000000000008 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.0000000021.00000000000000000009' <- 0000000021.00000000000000000009 2019-12-07T19:34:04.613+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 32 ==== PGlog(1.2s0 log log((0'0,23'10], crt=23'10) pi ([0,0] all_participants= intervals=) e27/27) v6 ==== 5145+0+0 (unknown 0 0 0) 0x558e84041000 con 0x558e818d6800 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.0000000021.00000000000000000010' <- 0000000021.00000000000000000010 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.0000000021.00000000000000000011' <- 0000000021.00000000000000000011 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.0000000021.00000000000000000012' <- 0000000021.00000000000000000012 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._biginfo' <- _biginfo 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._epoch' <- _epoch 2019-12-07T19:34:04.613+0000 7fe2e753b700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 MLogRec from 5(4) log log((0'0,23'10], crt=23'10) pi ([0,0] all_participants= intervals=) +create_info 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:04.613+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLogRec from 5(4) log log((0'0,23'10], crt=23'10) pi ([0,0] all_participants= intervals=) +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.missing/0000000000000001.B0E469FE.head.smithi13140796-28..' <- missing/0000000000000001.B0E469FE.head.smithi13140796-28.. 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.missing/0000000000000001.B376C33C.head.smithi13140796-50..' <- missing/0000000000000001.B376C33C.head.smithi13140796-50.. 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.missing/0000000000000001.BC8CBBB8.head.smithi13140796-49..' <- missing/0000000000000001.BC8CBBB8.head.smithi13140796-49.. 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.missing/0000000000000001.BD1A381A.head.smithi13140796-18..' <- missing/0000000000000001.BD1A381A.head.smithi13140796-18.. 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4d500 cost 13019 (1 ios * 4000 + 9019 bytes) 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4d500 onodes shared_blobs 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4d500 allocated 0x[] released 0x[] 2019-12-07T19:34:04.613+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 33 ==== pg_info2(1.2s0 1.2s0 1.2s3( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558e8122b800 con 0x558e818d6800 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 prepare 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4d500 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 io_done 2019-12-07T19:34:04.613+0000 7fe2e753b700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(3) info: 1.2s3( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 0s) 2019-12-07T19:34:04.613+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(3) info: 1.2s3( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 0s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.613+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 17 ==== pg_info2(1.2s0 1.2s0 1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558e8122b800 con 0x558e818fe000 2019-12-07T19:34:04.613+0000 7fe2d4a80700 3 osd.0 27 handle_osd_map epochs [27,27], i have 27, src has [1,27] 2019-12-07T19:34:04.613+0000 7fe2d4a80700 10 osd.0 27 no new maps here, dropping 2019-12-07T19:34:04.613+0000 7fe2e753b700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 MInfoRec from 7(4) info: 1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 0s) 2019-12-07T19:34:04.613+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 7(4) info: 1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 0s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.613+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 30 ==== osd_map(27..27 src has 1..27) v4 ==== 328+0+0 (unknown 0 0 0) 0x558e841a0fc0 con 0x558e818d6800 2019-12-07T19:34:04.613+0000 7fe2d4a80700 20 osd.0 27 OSD::ms_dispatch: osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.613+0000 7fe2d4a80700 10 osd.0 27 do_waiters -- start 2019-12-07T19:34:04.613+0000 7fe2d4a80700 10 osd.0 27 do_waiters -- finish 2019-12-07T19:34:04.613+0000 7fe2d4a80700 20 osd.0 27 _dispatch 0x558e841a0fc0 osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.613+0000 7fe2d4a80700 3 osd.0 27 handle_osd_map epochs [27,27], i have 27, src has [1,27] 2019-12-07T19:34:04.613+0000 7fe2d4a80700 10 osd.0 27 no new maps here, dropping 2019-12-07T19:34:04.613+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 18 ==== osd_map(27..27 src has 1..27) v4 ==== 328+0+0 (unknown 0 0 0) 0x558e8419e6c0 con 0x558e818fe000 2019-12-07T19:34:04.613+0000 7fe2d4a80700 20 osd.0 27 OSD::ms_dispatch: osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.613+0000 7fe2d4a80700 10 osd.0 27 do_waiters -- start 2019-12-07T19:34:04.613+0000 7fe2d4a80700 10 osd.0 27 do_waiters -- finish 2019-12-07T19:34:04.613+0000 7fe2d4a80700 20 osd.0 27 _dispatch 0x558e8419e6c0 osd_map(27..27 src has 1..27) v4 2019-12-07T19:34:04.613+0000 7fe2d4a80700 3 osd.0 27 handle_osd_map epochs [27,27], i have 27, src has [1,27] 2019-12-07T19:34:04.613+0000 7fe2d4a80700 10 osd.0 27 no new maps here, dropping 2019-12-07T19:34:04.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.613+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 71 ==== pg_query2(1.bs3 1.bs3 query(fulllog 0'0 epoch_sent 27) e27/27) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417d180 con 0x558e818fe400 2019-12-07T19:34:04.613+0000 7fe2e6d3a700 15 osd.0 27 enqueue_peering_evt 1.bs3 epoch_sent: 27 epoch_requested: 27 MQuery 1.bs3 from 4(0) query_epoch 27 query: query(fulllog 0'0 epoch_sent 27) 2019-12-07T19:34:04.613+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MQuery 1.bs3 from 4(0) query_epoch 27 query: query(fulllog 0'0 epoch_sent 27)) prio 255 cost 10 e27) 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MQuery 1.bs3 from 4(0) query_epoch 27 query: query(fulllog 0'0 epoch_sent 27)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MQuery 1.bs3 from 4(0) query_epoch 27 query: query(fulllog 0'0 epoch_sent 27)) prio 255 cost 10 e27) pg 0x558e81928000 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MQuery 1.bs3 from 4(0) query_epoch 27 query: query(fulllog 0'0 epoch_sent 27) 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] log request from 4(0) 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] sending info+missing+full log 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] sending log((0'0,21'12], crt=21'12) missing(0 may_include_deletes = 1) 2019-12-07T19:34:04.613+0000 7fe2c6263700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- PGlog(1.bs0 log log((0'0,21'12], crt=21'12) pi ([0,0] all_participants= intervals=) e27/27) v6 -- 0x558e811b8000 con 0x558e818fe400 2019-12-07T19:34:04.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLogRec from 5(4) log log((0'0,23'10], crt=23'10) pi ([0,0] all_participants= intervals=) +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLogRec from 5(4) log log((0'0,23'10], crt=23'10) pi ([0,0] all_participants= intervals=) +create_info) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.613+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:04.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MLogRec from 5(4) log log((0'0,23'10], crt=23'10) pi ([0,0] all_participants= intervals=) +create_info 2019-12-07T19:34:04.613+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] state: searching osd.5(4) log for unfound items 2019-12-07T19:34:04.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] proc_replica_log for osd.5(4): 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) log((0'0,23'10], crt=23'10) missing(0 may_include_deletes = 1) 2019-12-07T19:34:04.613+0000 7fe2caa6c700 10 proc_replica_log for osd.5(4): 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) log((0'0,23'10], crt=23'10) missing(0 may_include_deletes = 1) 2019-12-07T19:34:04.613+0000 7fe2caa6c700 10 proc_replica_log: osd.5(4) same log head, not looking for divergent objects 2019-12-07T19:34:04.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] peer osd.5(4) now 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) missing(0 may_include_deletes = 1) 2019-12-07T19:34:04.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=3},5={(1+0)=3}} ps=[2~1]] search_for_missing 1:41df217a:::smithi13140796-39:head 21'5 is on osd.5(4) 2019-12-07T19:34:04.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=2,(0+1)=1},5={(1+0)=3}} ps=[2~1]] search_for_missing 1:41fe8a11:::smithi13140796-45:head 21'6 is on osd.5(4) 2019-12-07T19:34:04.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+0)=1,(0+1)=2},5={(1+0)=3}} ps=[2~1]] search_for_missing 1:4986b721:::smithi13140796-43:head 21'4 is on osd.5(4) 2019-12-07T19:34:04.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] add_source_info 1:4c9e444c:::smithi13140796-48:head delete, ignoring source 2019-12-07T19:34:04.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] needs_recovery_map missing {1:41df217a:::smithi13140796-39:head=21'5 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:41fe8a11:::smithi13140796-45:head=21'6 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4986b721:::smithi13140796-43:head=21'4 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.614+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.614+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s4 1.2s4 1.2s0( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 -- 0x558e8404d100 con 0x558e818d6800 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(3) info: 1.2s3( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 0s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(3) info: 1.2s3( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 0s)) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.614+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(3) info: 1.2s3( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 0s) 2019-12-07T19:34:04.614+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] state: peer osd.5(3) activated and committed 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(4)=3} 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 4 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 4 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 8 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.614+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 27:41 2019-12-07T19:34:04.614+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 7(4) info: 1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 0s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 7(4) info: 1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 0s)) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.614+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 7(4) info: 1.2s4( v 23'10 lc 21'3 (0'0,23'10] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 0s) 2019-12-07T19:34:04.614+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] state: peer osd.7(4) activated and committed 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(4)=3} 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 4 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 4 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 8 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.614+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 27:42 2019-12-07T19:34:04.614+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.614+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 122 ==== pg_notify2(1.2s0 1.2s0 (query:27 sent:27 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e27/27) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e8404a700 con 0x558e818d7c00 2019-12-07T19:34:04.615+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 1(2) notify: (query:27 sent:27 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:04.615+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 1(2) notify: (query:27 sent:27 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) 2019-12-07T19:34:04.615+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 123 ==== pg_info2(1.as3 1.as3 1.as0( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404bc00 con 0x558e818d7c00 2019-12-07T19:34:04.615+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.as3 epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.066931508s int 16s) 2019-12-07T19:34:04.615+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.066931508s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 1(2) notify: (query:27 sent:27 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27)> waiting <> waiting_peering {} 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 1(2) notify: (query:27 sent:27 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MNotifyRec 1.2s0 from 1(2) notify: (query:27 sent:27 1.2s2( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2->0 ([18,25] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] state: Active: got notify from 1(2), already have info from that osd, ignoring 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.066931508s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.066931508s int 16s)) prio 255 cost 10 e27) pg 0x558e8193b000 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) pg_lease(ru 0s ub 44.066931508s int 16s) 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] state: got info from osd.1(0) 1.as0( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 44.066931508s int 16s) 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] peer_clock_delta_lb 0.002061620s -> ruub 44.064869888s 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] proc_lease readable_until_ub now 44.064869888s 2019-12-07T19:34:04.615+0000 7fe2c6a64700 5 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.615888+0000 6 0.000081 2019-12-07T19:34:04.615+0000 7fe2c6a64700 5 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.615+0000 7fe2c6a64700 5 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped NOTIFY mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped mbc={} ps=[2~1]] activate - no missing, moving last_complete 21'2 -> 21'2 2019-12-07T19:34:04.615+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 72 ==== pg_info2(1.bs3 1.bs3 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) e27/27) v1 ==== 1030+0+0 (unknown 0 0 0) 0x558e83ebd800 con 0x558e818fe400 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.615+0000 7fe2e6d3a700 15 osd.0 27 enqueue_peering_evt 1.bs3 epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.615+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26)) prio 255 cost 10 e27) 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e81a0b500 seq 14 2019-12-07T19:34:04.615+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x808d6 2019-12-07T19:34:04.615+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:04.615+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.615+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.615+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:04.615+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.615+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 73 ==== pg_info2(1.2s0 1.2s0 1.2s1( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558e83ebd800 con 0x558e818fe400 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:04.615+0000 7fe2e6d3a700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(1) info: 1.2s1( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s) 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b500 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:04.615+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(1) info: 1.2s1( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b500 onodes shared_blobs 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b500 allocated 0x[] released 0x[] 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 prepare 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b500 2019-12-07T19:34:04.615+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 io_done 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(1) info: 1.2s1( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(1) info: 1.2s1( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s)) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26)) prio 255 cost 10 e27) pg 0x558e81928000 2019-12-07T19:34:04.615+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(1) info: 1.2s1( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s) 2019-12-07T19:34:04.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recalc_readable_until peer osd.4 ruub 44.065221680s 2019-12-07T19:34:04.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] state: got info from osd.4(0) 1.bs0( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:04.615+0000 7fe2c6263700 5 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:04.616051+0000 7 0.000150 2019-12-07T19:34:04.615+0000 7fe2c6263700 5 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:34:04.615+0000 7fe2c6263700 5 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recalc_readable_until readable_until[_ub] 0s (sent 44.065221680s) 2019-12-07T19:34:04.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped NOTIFY mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:04.615+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] state: peer osd.4(1) activated and committed 2019-12-07T19:34:04.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped mbc={} ps=[2~1]] activate - no missing, moving last_complete 21'12 -> 21'12 2019-12-07T19:34:04.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.615+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:04.615+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:34:04.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e83d4c900 seq 28 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.615+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:04.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:04.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:04.615+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:04.615+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c900 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:04.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c900 onodes shared_blobs 2019-12-07T19:34:04.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c900 allocated 0x[] released 0x[] 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.615+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 prepare 2019-12-07T19:34:04.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c900 2019-12-07T19:34:04.615+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 io_done 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(4)=3} 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 4 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 4 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 8 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.615+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 27:43 2019-12-07T19:34:04.615+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.615+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.615+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.615+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (30) on file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]), flushing log 2019-12-07T19:34:04.615+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:04.615+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.615+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 30 len 0x49 crc 0x3dda4fb) 2019-12-07T19:34:04.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x1d000~1000 to file(ino 1 size 0x1d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.615+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x1d000 0x1d000~1000 to file(ino 1 size 0x1d000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.615+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x1e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.615+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1d000 2019-12-07T19:34:04.615+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x1e000 2019-12-07T19:34:04.615+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:04.615+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:34:04.615+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.615+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 34 ==== pg_info2(1.2s0 1.2s0 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=19/19/0 sis=26) e27/27) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558e83ebbc00 con 0x558e818d6800 2019-12-07T19:34:04.615+0000 7fe2e753b700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(4) info: 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=19/19/0 sis=26) pg_lease_ack(ruub 0s) 2019-12-07T19:34:04.615+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(4) info: 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=19/19/0 sis=26) pg_lease_ack(ruub 0s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.615+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 70 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558e81a58700 con 0x558e818d7000 2019-12-07T19:34:04.615+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.1s1 epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061862442s ub 44.061862442s int 16s) 2019-12-07T19:34:04.615+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061862442s ub 44.061862442s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061862442s ub 44.061862442s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061862442s ub 44.061862442s int 16s)) prio 255 cost 10 e27) pg 0x558e819e4000 2019-12-07T19:34:04.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.1s0( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061862442s ub 44.061862442s int 16s) 2019-12-07T19:34:04.616+0000 7fe2cb26d700 20 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 active+remapped mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.616+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:04.616+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:04.616+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e83c56c00 seq 17 2019-12-07T19:34:04.616+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:04.616+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:04.616+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:04.616+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56c00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.616+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56c00 onodes shared_blobs 2019-12-07T19:34:04.616+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56c00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.616+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 prepare 2019-12-07T19:34:04.616+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56c00 2019-12-07T19:34:04.616+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 io_done 2019-12-07T19:34:04.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(4) info: 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=19/19/0 sis=26) pg_lease_ack(ruub 0s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(4) info: 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=19/19/0 sis=26) pg_lease_ack(ruub 0s)) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 5(4) info: 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=19/19/0 sis=26) pg_lease_ack(ruub 0s) 2019-12-07T19:34:04.616+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore.onode(0x558e83cb3400).flush flush done 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkey_range remove range start: 0x0000000000000462'.0000000000.00000000000000000000' end: 0x0000000000000462'.4294967295.18446744073709551615' 2019-12-07T19:34:04.616+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkey_range 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.616+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.0000000021.00000000000000000001' <- 0000000021.00000000000000000001 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.0000000021.00000000000000000002' <- 0000000021.00000000000000000002 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.0000000021.00000000000000000003' <- 0000000021.00000000000000000003 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.0000000021.00000000000000000004' <- 0000000021.00000000000000000004 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.0000000021.00000000000000000005' <- 0000000021.00000000000000000005 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.0000000021.00000000000000000006' <- 0000000021.00000000000000000006 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.0000000023.00000000000000000007' <- 0000000023.00000000000000000007 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._biginfo' <- _biginfo 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._epoch' <- _epoch 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.missing/0000000000000001.51B7DFC8.head.smithi13140796-24..' <- missing/0000000000000001.51B7DFC8.head.smithi13140796-24.. 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.missing/0000000000000001.59B75E6C.head.smithi13140796-12..' <- missing/0000000000000001.59B75E6C.head.smithi13140796-12.. 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.missing/0000000000000001.5E6166C2.head.smithi13140796-35..' <- missing/0000000000000001.5E6166C2.head.smithi13140796-35.. 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.616+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.616+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1b00 cost 9084 (1 ios * 4000 + 5084 bytes) 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1b00 onodes shared_blobs 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1b00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.616+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 prepare 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1b00 2019-12-07T19:34:04.616+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 io_done 2019-12-07T19:34:04.616+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.616+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 19 ==== pg_lease(1.4s1 pg_lease(ru 36.052429711s ub 44.052553893s int 16s) e27/27) v1 ==== 58+0+0 (unknown 0 0 0) 0x558e84053000 con 0x558e818fe000 2019-12-07T19:34:04.616+0000 7fe2e753b700 15 osd.0 27 enqueue_peering_evt 1.4s1 epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.7 pg_lease(ru 36.052429711s ub 44.052553893s int 16s) 2019-12-07T19:34:04.616+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.7 pg_lease(ru 36.052429711s ub 44.052553893s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.616+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.616+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.7 pg_lease(ru 36.052429711s ub 44.052553893s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.616+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.616+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.7 pg_lease(ru 36.052429711s ub 44.052553893s int 16s)) prio 255 cost 10 e27) pg 0x558e81938000 2019-12-07T19:34:04.616+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 27 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.7 pg_lease(ru 36.052429711s ub 44.052553893s int 16s) 2019-12-07T19:34:04.616+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 27 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] proc_lease pg_lease(ru 36.052429711s ub 44.052553893s int 16s) 2019-12-07T19:34:04.616+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 27 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] peer_clock_delta_ub 18446744073.690501011s -> ru 36.071480316s 2019-12-07T19:34:04.616+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 27 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] proc_lease readable_until now 36.071480316s 2019-12-07T19:34:04.616+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 27 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] peer_clock_delta_lb 18446744073.689810581s -> ruub 44.072294928s 2019-12-07T19:34:04.616+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 27 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] proc_lease readable_until_ub now 44.072294928s 2019-12-07T19:34:04.616+0000 7fe2c9a6a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_lease_ack(1.4s0 pg_lease_ack(ruub 44.052553893s) e27/27) v1 -- 0x558e84199860 con 0x558e818fe000 2019-12-07T19:34:04.616+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:04.616+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.002571 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.618+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 26 ==== pg_lease(1.ds5 pg_lease(ru 36.088857038s ub 44.090670001s int 16s) e27/27) v1 ==== 58+0+0 (unknown 0 0 0) 0x558e84053000 con 0x558e818d6000 2019-12-07T19:34:04.618+0000 7fe2e753b700 15 osd.0 27 enqueue_peering_evt 1.ds5 epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.2 pg_lease(ru 36.088857038s ub 44.090670001s int 16s) 2019-12-07T19:34:04.618+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.2 pg_lease(ru 36.088857038s ub 44.090670001s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 30 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.2 pg_lease(ru 36.088857038s ub 44.090670001s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.2 pg_lease(ru 36.088857038s ub 44.090670001s int 16s)) prio 255 cost 10 e27) pg 0x558e819e7000 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 3 cleaned 0 in 0.006636207s (0.000000112s flush + 0.006636095s kv commit) 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 7 submitting 7 deferred done 0 stable 0 2019-12-07T19:34:04.618+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MLease epoch 27 from osd.2 pg_lease(ru 36.088857038s ub 44.090670001s int 16s) 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1418 < min_flush_size 524288 2019-12-07T19:34:04.618+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] proc_lease pg_lease(ru 36.088857038s ub 44.090670001s int 16s) 2019-12-07T19:34:04.618+0000 7fe2c5261700 20 osd.0 pg_epoch: 27 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] peer_clock_delta_ub 0.017223111s -> ru 36.071633927s 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:04.618+0000 7fe2c5261700 20 osd.0 pg_epoch: 27 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] proc_lease readable_until now 36.071633927s 2019-12-07T19:34:04.618+0000 7fe2c5261700 20 osd.0 pg_epoch: 27 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] peer_clock_delta_lb 0.016890965s -> ruub 44.073779036s 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8348 < min_flush_size 524288 2019-12-07T19:34:04.618+0000 7fe2c5261700 20 osd.0 pg_epoch: 27 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] proc_lease readable_until_ub now 44.073779036s 2019-12-07T19:34:04.618+0000 7fe2c5261700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease_ack(1.ds0 pg_lease_ack(ruub 44.090670001s) e27/27) v1 -- 0x558e8403e780 con 0x558e818d6000 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:04.618+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 17189 < min_flush_size 524288 2019-12-07T19:34:04.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8106c000,0x558e83c56f00,0x558e8417b200> 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 kv_submitted 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c000 2019-12-07T19:34:04.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 finishing 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c000 onodes 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c000 done 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c000 [] 2019-12-07T19:34:04.618+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 18480 < min_flush_size 524288 2019-12-07T19:34:04.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 kv_submitted 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56f00 2019-12-07T19:34:04.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 finishing 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56f00 onodes 0x558e8402ec80 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56f00 done 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56600 kv_submitted 2019-12-07T19:34:04.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56f00 [] 2019-12-07T19:34:04.618+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 19771 < min_flush_size 524288 2019-12-07T19:34:04.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 kv_submitted 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417b200 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:34:04.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 finishing 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417b200 onodes 0x558e83cb3400 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b200 done 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1b00 kv_queued 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 20794 < min_flush_size 524288 2019-12-07T19:34:04.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417b200 [] 2019-12-07T19:34:04.618+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:04.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 25659 < min_flush_size 524288 2019-12-07T19:34:04.618+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:04.618+0000 7fe2cba6e700 15 osd.0 27 enqueue_peering_evt 1.8s5 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.618+0000 7fe2cba6e700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 25682 < min_flush_size 524288 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 25682 < min_flush_size 524288 2019-12-07T19:34:04.618+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.618+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.618+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x808d6~6452 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.618+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558e841a4000 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x808d6 0x808d6~6452 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x508d6 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x8d6 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd28 and padding block with 0x2d8 2019-12-07T19:34:04.618+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 23'14 lc 21'3 (0'0,23'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'14 lcod 0'0 unknown m=6 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.618+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 23'14 lc 21'3 (0'0,23'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'14 lcod 0'0 unknown m=6 mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.618+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:04.618+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.8s0 1.8s0 1.8s5( v 23'14 lc 21'3 (0'0,23'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e83ebd100 con 0x558e818d7c00 2019-12-07T19:34:04.618+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.618+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.618+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x86d28 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.618+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 7 cleaned 0 in 0.000473203s (0.000000123s flush + 0.000473080s kv commit) 2019-12-07T19:34:04.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fb500,0x558e83c56600,0x558e83d4d500,0x558e81a0b500,0x558e83d4c900,0x558e83c56c00,0x558e819a1b00> 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 kv_submitted 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fb500 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 finishing 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fb500 onodes 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb500 done 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fb500 [] 2019-12-07T19:34:04.619+0000 7fe2cca70700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 18 epoch_requested: 18 RenewLease 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.619+0000 7fe2cca70700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 RenewLease) prio 255 cost 10 e18) 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 kv_submitted 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56600 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 finishing 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56600 onodes 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56600 done 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56600 [] 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.619+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 kv_submitted 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4d500 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 finishing 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4d500 onodes 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4d500 done 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4d500 [] 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 kv_submitted 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b500 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 finishing 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b500 onodes 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b500 done 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b500 [] 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 kv_submitted 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c900 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 finishing 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c900 onodes 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c900 done 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c900 [] 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 kv_submitted 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56c00 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 finishing 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56c00 onodes 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56c00 done 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56c00 [] 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 kv_submitted 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1b00 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 finishing 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1b00 onodes 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1b00 done 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1b00 [] 2019-12-07T19:34:04.619+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.619+0000 7fe2ca26b700 15 osd.0 27 enqueue_peering_evt 1.3s5 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.619+0000 7fe2c9269700 15 osd.0 27 enqueue_peering_evt 1.5s4 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.619+0000 7fe2c9269700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.619+0000 7fe2ca26b700 15 osd.0 27 enqueue_peering_evt 1.bs1 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.619+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.619+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.619+0000 7fe2ca26b700 15 osd.0 27 enqueue_peering_evt 1.bs3 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 19 processed at 27 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 19 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.619+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558e83d07000 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'7 lcod 0'0 unknown m=3 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558e81231000 2019-12-07T19:34:04.619+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'7 lcod 0'0 unknown m=3 mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.619+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 lcod 0'0 unknown m=3 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.619+0000 7fe2c9269700 10 osd.0 27 send_incremental_map 26 -> 27 to 0x558e818d6000 v2:172.21.15.131:6804/37077 2019-12-07T19:34:04.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=23'10 lcod 0'0 unknown m=3 mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.619+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(27..27 src has 1..27) v4 -- 0x558e841a0d80 con 0x558e818d6000 2019-12-07T19:34:04.619+0000 7fe2c9269700 10 osd.0 27 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 26 -> 27 (shared) 2019-12-07T19:34:04.619+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.3s0 1.3s0 1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e83eba000 con 0x558e818fe400 2019-12-07T19:34:04.619+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.5s0 1.5s0 1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e83c46700 con 0x558e818d6000 2019-12-07T19:34:04.619+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558e83c78000 2019-12-07T19:34:04.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 lcod 0'0 unknown m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 lcod 0'0 unknown m=4 mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.619+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.bs0 1.bs0 1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e83c9d500 con 0x558e818fe400 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 19 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 19 processed at 27) prio 255 cost 10 e27) pg 0x558e81928000 2019-12-07T19:34:04.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 19 processed at 27 2019-12-07T19:34:04.619+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 crt=21'12 remapped mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.619+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.bs0 1.bs0 1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=19/19/0 sis=26) e27/27) v1 -- 0x558e83c9f800 con 0x558e818fe400 2019-12-07T19:34:04.619+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 RenewLease) prio 255 cost 10 e18) queued 2019-12-07T19:34:04.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 18 epoch_requested: 18 RenewLease) prio 255 cost 10 e18) pg 0x558e81075000 2019-12-07T19:34:04.619+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 18 epoch_requested: 18 RenewLease 2019-12-07T19:34:04.619+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] old_peering_msg reply_epoch 18 query_epoch 18 last_peering_reset 26 2019-12-07T19:34:04.619+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] discard old epoch_sent: 18 epoch_requested: 18 RenewLease 2019-12-07T19:34:04.619+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7fe2caa6c700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.619+0000 7fe2caa6c700 15 osd.0 27 enqueue_peering_evt 1.as3 epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) 2019-12-07T19:34:04.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.619+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.619+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] state: _activate_committed 27 peer_activated now 0(0),4(1),5(3),7(4) last_interval_started 18 last_epoch_started 19 same_interval_since 26 2019-12-07T19:34:04.619+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27) prio 255 cost 10 e27) pg 0x558e8193b000 2019-12-07T19:34:04.619+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 ActivateCommitted from 27 processed at 27 2019-12-07T19:34:04.619+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 crt=21'2 remapped mbc={} ps=[2~1]] state: react 27 telling primary 2019-12-07T19:34:04.619+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.as0 1.as0 1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e8404d800 con 0x558e818d7c00 2019-12-07T19:34:04.619+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 124 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558e81a58000 con 0x558e818d7c00 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.es4 epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063058723s ub 44.063058723s int 16s) 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063058723s ub 44.063058723s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063058723s ub 44.063058723s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 125 ==== pg_info2(1.7s3 1.7s3 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558e83ebd100 con 0x558e818d7c00 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063058723s ub 44.063058723s int 16s)) prio 255 cost 10 e27) pg 0x558e81a98000 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.7s3 epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063255566s ub 44.063255566s int 16s) 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063255566s ub 44.063255566s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.619+0000 7fe2c4a60700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063058723s ub 44.063058723s int 16s) 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.619+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 126 ==== pg_info2(1.9s3 1.9s3 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558e83ebd100 con 0x558e818d7c00 2019-12-07T19:34:04.619+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e83c56000 seq 19 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.9s3 epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062707990s ub 44.062707990s int 16s) 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062707990s ub 44.062707990s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.619+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:04.619+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 127 ==== MRecoveryReserve(1.7s3 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e817b4960 con 0x558e818d7c00 2019-12-07T19:34:04.619+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56000 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56000 onodes shared_blobs 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56000 allocated 0x[] released 0x[] 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.7s3 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.619+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 prepare 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56000 2019-12-07T19:34:04.619+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 io_done 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 128 ==== MRecoveryReserve(1.es4 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e817b4960 con 0x558e818d7c00 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.es4 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558e81a98000 2019-12-07T19:34:04.619+0000 7fe2c4a60700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 71 ==== pg_info2(1.fs1 1.fs1 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558e83ebd100 con 0x558e818d7000 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.fs1 epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061994571s ub 44.061994571s int 16s) 2019-12-07T19:34:04.619+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061994571s ub 44.061994571s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.619+0000 7fe2c4a60700 5 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.620602+0000 4 0.000242 2019-12-07T19:34:04.619+0000 7fe2c4a60700 5 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.619+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.619+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062707990s ub 44.062707990s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.619+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062707990s ub 44.062707990s int 16s)) prio 255 cost 10 e27) pg 0x558e8192b000 2019-12-07T19:34:04.619+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063255566s ub 44.063255566s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.619+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.619+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063255566s ub 44.063255566s int 16s)) prio 255 cost 10 e27) pg 0x558e8122e000 2019-12-07T19:34:04.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.9s0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.062707990s ub 44.062707990s int 16s) 2019-12-07T19:34:04.619+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.7s0( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.063255566s ub 44.063255566s int 16s) 2019-12-07T19:34:04.619+0000 7fe2c7265700 20 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.619+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7fe2c8267700 20 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:04.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e8417ac00 seq 19 2019-12-07T19:34:04.619+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:04.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:04.619+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:04.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:04.619+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e819a1200 seq 28 2019-12-07T19:34:04.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:04.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417ac00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.619+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:04.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417ac00 onodes shared_blobs 2019-12-07T19:34:04.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.619+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:04.620+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:04.620+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes shared_blobs 2019-12-07T19:34:04.620+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 prepare 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[] released 0x[] 2019-12-07T19:34:04.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417ac00 2019-12-07T19:34:04.620+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 io_done 2019-12-07T19:34:04.620+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:34:04.620+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558e8122e000 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2046 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3069 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3092 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3092 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2c8267700 5 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.620730+0000 4 0.000091 2019-12-07T19:34:04.620+0000 7fe2cea74700 15 osd.0 27 enqueue_peering_evt 1.es4 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2cea74700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x86d28~c14 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2c8267700 5 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.620+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x86d28 0x86d28~c14 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x56d28 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd28 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x93c and padding block with 0x6c4 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061994571s ub 44.061994571s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061994571s ub 44.061994571s int 16s)) prio 255 cost 10 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.620+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(0) info: 1.fs0( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061994571s ub 44.061994571s int 16s) 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.620+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:04.620+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:04.620+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e819a0c00 seq 23 2019-12-07T19:34:04.620+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.620+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:04.620+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.620+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:04.620+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e81a98000 2019-12-07T19:34:04.620+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0c00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0c00 onodes shared_blobs 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.620+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 prepare 2019-12-07T19:34:04.620+0000 7fe2c4a60700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0c00 2019-12-07T19:34:04.620+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 io_done 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7fe2c4a60700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.es0 GRANT e27/27) v3 -- 0x558e84199860 con 0x558e818d7c00 2019-12-07T19:34:04.620+0000 7fe2c4a60700 5 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.620830+0000 1 0.000033 2019-12-07T19:34:04.620+0000 7fe2c4a60700 5 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.620+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:04.620+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x8793c 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 3 cleaned 0 in 0.000159509s (0.000000082s flush + 0.000159427s kv commit) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x8793c~416 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x8793c 0x8793c~416 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x5793c 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x93c 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd52 and padding block with 0x2ae 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56000,0x558e8417ac00,0x558e819a1200> 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 kv_submitted 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56000 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 finishing 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56000 onodes 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56000 done 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56000 [] 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 kv_submitted 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417ac00 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 finishing 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417ac00 onodes 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417ac00 done 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417ac00 [] 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.620+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x87d52 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000410436s (0.000000077s flush + 0.000410359s kv commit) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a0c00> 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 kv_submitted 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0c00 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 finishing 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0c00 onodes 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0c00 done 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0c00 [] 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.620+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 74 ==== pg_info2(1.3s5 1.3s5 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558e83c9f800 con 0x558e818fe400 2019-12-07T19:34:04.620+0000 7fe2e6d3a700 15 osd.0 27 enqueue_peering_evt 1.3s5 epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057306578s ub 44.057306578s int 16s) 2019-12-07T19:34:04.620+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057306578s ub 44.057306578s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.620+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.620+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057306578s ub 44.057306578s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.620+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.620+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057306578s ub 44.057306578s int 16s)) prio 255 cost 10 e27) pg 0x558e81231000 2019-12-07T19:34:04.620+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057306578s ub 44.057306578s int 16s) 2019-12-07T19:34:04.620+0000 7fe2ca26b700 20 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.620+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:04.620+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:04.620+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e83c56900 seq 3 2019-12-07T19:34:04.620+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:04.620+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:04.620+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:04.620+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56900 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.620+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56900 onodes shared_blobs 2019-12-07T19:34:04.620+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56900 allocated 0x[] released 0x[] 2019-12-07T19:34:04.620+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 prepare 2019-12-07T19:34:04.620+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56900 2019-12-07T19:34:04.620+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 io_done 2019-12-07T19:34:04.620+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 686 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1034 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:04.620+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 75 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558e83c9f800 con 0x558e818fe400 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1057 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1057 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x87d52~421 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2e6d3a700 15 osd.0 27 enqueue_peering_evt 1.bs1 epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057430495s ub 44.057430495s int 16s) 2019-12-07T19:34:04.620+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057430495s ub 44.057430495s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x87d52 0x87d52~421 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x57d52 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd52 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x173 and padding block with 0xe8d 2019-12-07T19:34:04.620+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.620+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057430495s ub 44.057430495s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.620+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.620+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057430495s ub 44.057430495s int 16s)) prio 255 cost 10 e27) pg 0x558e83c78000 2019-12-07T19:34:04.620+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.057430495s ub 44.057430495s int 16s) 2019-12-07T19:34:04.620+0000 7fe2c6263700 20 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.620+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:04.620+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:04.620+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e83d4c600 seq 3 2019-12-07T19:34:04.620+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.620+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:04.620+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.620+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c600 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.620+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c600 onodes shared_blobs 2019-12-07T19:34:04.620+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.620+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c600 prepare 2019-12-07T19:34:04.620+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c600 2019-12-07T19:34:04.620+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c600 io_done 2019-12-07T19:34:04.620+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x88173 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000163182s (0.000000082s flush + 0.000163100s kv commit) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56900> 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 kv_submitted 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56900 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 finishing 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56900 onodes 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56900 done 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56900 [] 2019-12-07T19:34:04.620+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x88173~416 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.620+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x88173 0x88173~416 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x58173 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x173 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.620+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x589 and padding block with 0xa77 2019-12-07T19:34:04.621+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x88589 2019-12-07T19:34:04.621+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.621+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.621+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.621+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.621+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000140815s (0.000000078s flush + 0.000140737s kv commit) 2019-12-07T19:34:04.621+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4c600> 2019-12-07T19:34:04.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.621+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c600 kv_submitted 2019-12-07T19:34:04.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c600 2019-12-07T19:34:04.621+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c600 finishing 2019-12-07T19:34:04.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c600 onodes 2019-12-07T19:34:04.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c600 done 2019-12-07T19:34:04.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:04.621+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c600 [] 2019-12-07T19:34:04.621+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.621+0000 7fe2cea74700 15 osd.0 27 enqueue_peering_evt 1.7s3 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.621+0000 7fe2cea74700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.621+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.621+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.621+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.621+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.621+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.621+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e8122e000 2019-12-07T19:34:04.621+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.621+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.7s0 GRANT e27/27) v3 -- 0x558e80231e00 con 0x558e818d7c00 2019-12-07T19:34:04.621+0000 7fe2c425f700 5 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.622608+0000 1 0.000024 2019-12-07T19:34:04.621+0000 7fe2c425f700 5 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.621+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:04.621+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.623+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.623+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 72 ==== MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=50, reqid=client.4540.0:214, at_version=27'5, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 167256+0+0 (unknown 0 0 0) 0x558e83e57800 con 0x558e818d7000 2019-12-07T19:34:04.623+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83b0b600 prio 127 cost 0 latency 0.000046 epoch 27 MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=50, reqid=client.4540.0:214, at_version=27'5, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.623+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=50, reqid=client.4540.0:214, at_version=27'5, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.623+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 73 ==== MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=49, reqid=client.4540.0:205, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 163160+0+0 (unknown 0 0 0) 0x558e819d1800 con 0x558e818d7000 2019-12-07T19:34:04.623+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83b0be40 prio 127 cost 0 latency 0.000032 epoch 27 MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=49, reqid=client.4540.0:205, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.623+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=49, reqid=client.4540.0:205, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.623+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 129 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 23'14 (0'0,23'14] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558e8404d800 con 0x558e818d7c00 2019-12-07T19:34:04.623+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.8s5 epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.8s0( v 23'14 (0'0,23'14] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061823309s ub 44.061823309s int 16s) 2019-12-07T19:34:04.623+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.8s0( v 23'14 (0'0,23'14] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061823309s ub 44.061823309s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.623+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 130 ==== MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=99, to_read={1:ec1285ab:::smithi13140796-19:head=0,77824,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8417cc80 con 0x558e818d7c00 2019-12-07T19:34:04.623+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84176420 prio 127 cost 0 latency 0.000017 epoch 27 MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=99, to_read={1:ec1285ab:::smithi13140796-19:head=0,77824,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.623+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=99, to_read={1:ec1285ab:::smithi13140796-19:head=0,77824,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.623+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.623+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.8s0( v 23'14 (0'0,23'14] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061823309s ub 44.061823309s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.623+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.623+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.8s0( v 23'14 (0'0,23'14] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061823309s ub 44.061823309s int 16s)) prio 255 cost 10 e27) pg 0x558e841a4000 2019-12-07T19:34:04.623+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 23'14 lc 21'3 (0'0,23'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'14 lcod 0'0 active m=6 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.8s0( v 23'14 (0'0,23'14] local-lis/les=26/27 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.061823309s ub 44.061823309s int 16s) 2019-12-07T19:34:04.623+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 23'14 lc 21'3 (0'0,23'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'14 lcod 0'0 active m=6 mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.623+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=50, reqid=client.4540.0:214, at_version=27'5, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.623+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=50, reqid=client.4540.0:214, at_version=27'5, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) pg 0x558e819e4000 2019-12-07T19:34:04.623+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e81a0a900 seq 3 2019-12-07T19:34:04.623+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.623+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.623+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.623+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e83b0b600 prio 127 cost 0 latency 0.000420 MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=50, reqid=client.4540.0:214, at_version=27'5, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.623+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a900 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.623+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a900 onodes shared_blobs 2019-12-07T19:34:04.623+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a900 allocated 0x[] released 0x[] 2019-12-07T19:34:04.623+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=50, reqid=client.4540.0:214, at_version=27'5, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.623+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 prepare 2019-12-07T19:34:04.623+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a900 2019-12-07T19:34:04.623+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 io_done 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 snap_mapper.add_oid 1:891f0d13:::smithi13140796-14:4 4 2019-12-07T19:34:04.623+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.623+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.623+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=49, reqid=client.4540.0:205, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 snap_mapper.get_snaps 1:891f0d13:::smithi13140796-14:4 got.empty() 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 snap_mapper.set_snaps 1:891f0d13:::smithi13140796-14:4 4 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=49, reqid=client.4540.0:205, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.198F0B8C.4.smithi13140796-14.. 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 snap_mapper.add_oid set SNA_1_0000000000000004_.1_0000000000000001.198F0B8C.4.smithi13140796-14.. 2019-12-07T19:34:04.623+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e83b0be40 prio 127 cost 0 latency 0.000295 MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=49, reqid=client.4540.0:205, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.623+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=49, reqid=client.4540.0:205, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 snap_mapper.add_oid 1:f7919c75:::smithi13140796-20:4 4 2019-12-07T19:34:04.623+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 21'3 (0'0,21'3] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 active+remapped mbc={} ps=[2~1]] append_log log((0'0,21'3], crt=21'3) [27'4 (21'3) clone 1:891f0d13:::smithi13140796-14:4 by unknown.0.0:0 2019-12-07T19:33:58.819566+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,27'5 (21'3) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:214 2019-12-07T19:34:01.667796+0000 0 ObjectCleanRegions clean_offsets: [0~229376,884736~18446744073708666879], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.623+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.623+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 snap_mapper.get_snaps 1:f7919c75:::smithi13140796-20:4 got.empty() 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 snap_mapper.set_snaps 1:f7919c75:::smithi13140796-20:4 4 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.FE9893EA.4.smithi13140796-20.. 2019-12-07T19:34:04.623+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'4 (0'0,27'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active+remapped mbc={} ps=[2~1]] add_log_entry 27'4 (21'3) clone 1:891f0d13:::smithi13140796-14:4 by unknown.0.0:0 2019-12-07T19:33:58.819566+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 snap_mapper.add_oid set SNA_1_0000000000000004_.1_0000000000000001.FE9893EA.4.smithi13140796-20.. 2019-12-07T19:34:04.623+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'5 (0'0,27'5] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active+remapped mbc={} ps=[2~1]] add_log_entry 27'5 (21'3) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:214 2019-12-07T19:34:01.667796+0000 0 ObjectCleanRegions clean_offsets: [0~229376,884736~18446744073708666879], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.623+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'5 (0'0,27'5] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 5 2019-12-07T19:34:04.623+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'5 (0'0,27'5] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.623+0000 7fe2cb26d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.623+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 21'6 (0'0,21'6] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] append_log log((0'0,21'6], crt=21'6) [27'7 (21'2) clone 1:f7919c75:::smithi13140796-20:4 by unknown.0.0:0 2019-12-07T19:33:58.665983+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,27'8 (21'2) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:205 2019-12-07T19:34:01.660134+0000 0 ObjectCleanRegions clean_offsets: [0~327680,966656~18446744073708584959], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.623+0000 7fe2cb26d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'4, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.623+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] add_log_entry 27'7 (21'2) clone 1:f7919c75:::smithi13140796-20:4 by unknown.0.0:0 2019-12-07T19:33:58.665983+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.623+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] add_log_entry 27'8 (21'2) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:205 2019-12-07T19:34:01.660134+0000 0 ObjectCleanRegions clean_offsets: [0~327680,966656~18446744073708584959], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.623+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 8 2019-12-07T19:34:04.623+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:04.623+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e83d4c000 seq 18 2019-12-07T19:34:04.623+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.623+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) get_onode oid 1#1:891f0d13:::smithi13140796-14:4# key 0x818000000000000001891f0d1321736d'ithi13140796-14!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) r -2 v.len 0 2019-12-07T19:34:04.623+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e8417b800 seq 24 2019-12-07T19:34:04.623+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _clone 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# -> 1#1:891f0d13:::smithi13140796-14:4# 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1123 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.onode(0x558e811dd680).flush flush done 2019-12-07T19:34:04.623+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x0 2019-12-07T19:34:04.623+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_clone_range 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# -> 1#1:891f0d13:::smithi13140796-14:4# 0x0~e000 -> 0x0~e000 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) get_onode oid 1#1:f7919c75:::smithi13140796-20:4# key 0x818000000000000001f7919c'u!smithi13140796-20!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup src 0x0~2000: 0x0~2000 Blob(0x558e83c65730 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 sbid 0x0)) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10241 2019-12-07T19:34:04.623+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) make_blob_shared Blob(0x558e83c65730 blob([0x5c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 sbid 0x0)) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) make_blob_shared now Blob(0x558e83c65730 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=1)))) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup new Blob(0x558e83e6d5e0 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.blob(0x558e83e6d5e0) get_ref 0x0~2000 Blob(0x558e83e6d5e0 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.blob(0x558e83e6d5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup dst 0x0~2000: 0x0~2000 Blob(0x558e83e6d5e0 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup src 0x2000~2000: 0x0~2000 Blob(0x558e83c64e00 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 sbid 0x0)) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10242 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) r -2 v.len 0 2019-12-07T19:34:04.623+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) make_blob_shared Blob(0x558e83c64e00 blob([0x5e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 sbid 0x0)) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) make_blob_shared now Blob(0x558e83c64e00 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=1)))) 2019-12-07T19:34:04.623+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _clone 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# -> 1#1:f7919c75:::smithi13140796-20:4# 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup new Blob(0x558e83e6d570 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1124 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 bluestore.onode(0x558e83c72500).flush flush done 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.blob(0x558e83e6d570) get_ref 0x0~2000 Blob(0x558e83e6d570 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.blob(0x558e83e6d570) get_ref init 0x2000, 1000 2019-12-07T19:34:04.623+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x0 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup dst 0x2000~2000: 0x0~2000 Blob(0x558e83e6d570 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) 2019-12-07T19:34:04.623+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_clone_range 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# -> 1#1:f7919c75:::smithi13140796-20:4# 0x0~14000 -> 0x0~14000 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup src 0x4000~2000: 0x0~2000 Blob(0x558e83c65810 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 sbid 0x0)) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10243 2019-12-07T19:34:04.623+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) make_blob_shared Blob(0x558e83c65810 blob([0x60000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 sbid 0x0)) 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup src 0x0~2000: 0x0~2000 Blob(0x558e83ac49a0 blob([0x9e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 sbid 0x0)) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) make_blob_shared now Blob(0x558e83c65810 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=1)))) 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10244 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup new Blob(0x558e83f95d50 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) 2019-12-07T19:34:04.623+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared Blob(0x558e83ac49a0 blob([0x9e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 sbid 0x0)) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.blob(0x558e83f95d50) get_ref 0x0~2000 Blob(0x558e83f95d50 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.blob(0x558e83f95d50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared now Blob(0x558e83ac49a0 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=1)))) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup dst 0x4000~2000: 0x0~2000 Blob(0x558e83f95d50 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup new Blob(0x558e83f95650 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) 2019-12-07T19:34:04.623+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup src 0x6000~2000: 0x0~2000 Blob(0x558e817e9c00 blob([0x62000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10245 2019-12-07T19:34:04.623+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95650) get_ref 0x0~2000 Blob(0x558e83f95650 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) make_blob_shared Blob(0x558e817e9c00 blob([0x62000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95650) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) make_blob_shared now Blob(0x558e817e9c00 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=1)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup dst 0x0~2000: 0x0~2000 Blob(0x558e83f95650 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup new Blob(0x558e84014070 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup src 0x2000~2000: 0x0~2000 Blob(0x558e81866af0 blob([0xa0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10246 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e84014070) get_ref 0x0~2000 Blob(0x558e84014070 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e84014070) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared Blob(0x558e81866af0 blob([0xa0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup dst 0x6000~2000: 0x0~2000 Blob(0x558e84014070 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared now Blob(0x558e81866af0 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=1)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup src 0x8000~2000: 0x0~2000 Blob(0x558e817e9420 blob([0x64000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10247 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup new Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) make_blob_shared Blob(0x558e817e9420 blob([0x64000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e811d0a10) get_ref 0x0~2000 Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e811d0a10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) make_blob_shared now Blob(0x558e817e9420 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=1)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup dst 0x2000~2000: 0x0~2000 Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup new Blob(0x558e84014380 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup src 0x4000~2000: 0x0~2000 Blob(0x558e8194f570 blob([0xa2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10248 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e84014380) get_ref 0x0~2000 Blob(0x558e84014380 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared Blob(0x558e8194f570 blob([0xa2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e84014380) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared now Blob(0x558e8194f570 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=1)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup dst 0x8000~2000: 0x0~2000 Blob(0x558e84014380 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup new Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup src 0xa000~2000: 0x0~2000 Blob(0x558e817e95e0 blob([0x66000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10249 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e841a2070) get_ref 0x0~2000 Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) make_blob_shared Blob(0x558e817e95e0 blob([0x66000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e841a2070) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup dst 0x4000~2000: 0x0~2000 Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) make_blob_shared now Blob(0x558e817e95e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=1)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup src 0x6000~2000: 0x0~2000 Blob(0x558e83c65b90 blob([0xa4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup new Blob(0x558e840140e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10250 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared Blob(0x558e83c65b90 blob([0xa4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e840140e0) get_ref 0x0~2000 Blob(0x558e840140e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e840140e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared now Blob(0x558e83c65b90 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=1)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup dst 0xa000~2000: 0x0~2000 Blob(0x558e840140e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup new Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup src 0xc000~2000: 0x0~2000 Blob(0x558e817e8850 blob([0x68000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10251 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f955e0) get_ref 0x0~2000 Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) make_blob_shared Blob(0x558e817e8850 blob([0x68000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f955e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) make_blob_shared now Blob(0x558e817e8850 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=1)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup dst 0x6000~2000: 0x0~2000 Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup src 0x8000~2000: 0x0~2000 Blob(0x558e83c65a40 blob([0xa6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup new Blob(0x558e84014230 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10252 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared Blob(0x558e83c65a40 blob([0xa6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e84014230) get_ref 0x0~2000 Blob(0x558e84014230 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e84014230) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared now Blob(0x558e83c65a40 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=1)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dup dst 0xc000~2000: 0x0~2000 Blob(0x558e84014230 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dirty_range mark inline shard dirty 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup new Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e8417cb08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95570) get_ref 0x0~2000 Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95570) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup dst 0x8000~2000: 0x0~2000 Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup src 0xa000~2000: 0x0~2000 Blob(0x558e83c64d20 blob([0xa8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10253 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared Blob(0x558e83c64d20 blob([0xa8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _clone 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# -> 1#1:891f0d13:::smithi13140796-14:4# = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared now Blob(0x558e83c64d20 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=1)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rmattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# snapset 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup new Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95490) get_ref 0x0~2000 Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rmattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# snapset = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95490) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup dst 0xa000~2000: 0x0~2000 Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 1 keys 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup src 0xc000~2000: 0x0~2000 Blob(0x558e817e8d90 blob([0xaa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10254 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 1 keys = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared Blob(0x558e817e8d90 blob([0xaa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# hinfo_key (42 bytes) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared now Blob(0x558e817e8d90 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=1)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup new Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95420) get_ref 0x0~2000 Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2 keys 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95420) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup dst 0xc000~2000: 0x0~2000 Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2 keys = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup src 0xe000~2000: 0x0~2000 Blob(0x558e83af9340 blob([0xac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10255 2019-12-07T19:34:04.624+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared Blob(0x558e83af9340 blob([0xac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared now Blob(0x558e83af9340 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=1)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xe000~28000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup new Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 1#1:891f0d13:::smithi13140796-14:head# 0xe000~28000 - have 0xe000 (57344) bytes fadvise_flags 0x0 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f953b0) get_ref 0x0~2000 Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f953b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0xe000~28000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup dst 0xe000~2000: 0x0~2000 Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup src 0x10000~2000: 0x0~2000 Blob(0x558e83af9420 blob([0xae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10256 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared Blob(0x558e83af9420 blob([0xae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared now Blob(0x558e83af9420 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=1)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup new Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95340) get_ref 0x0~2000 Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95340) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup dst 0x10000~2000: 0x0~2000 Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup src 0x12000~2000: 0x0~2000 Blob(0x558e83af9500 blob([0xb0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10257 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared Blob(0x558e83af9500 blob([0xb0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83d4c000 20 blobs 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) make_blob_shared now Blob(0x558e83af9500 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=1)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup new Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f952d0) get_ref 0x0~2000 Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f952d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dup dst 0x12000~2000: 0x0~2000 Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dirty_range mark inline shard dirty 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72388) dirty_range mark inline shard dirty 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _clone 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# -> 1#1:f7919c75:::smithi13140796-20:4# = 0 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rmattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# snapset 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rmattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# snapset = 0 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 1 keys 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 1 keys = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# hinfo_key (42 bytes) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 2 keys 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 2 keys = 0 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x14000~27000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 1#1:f7919c75:::smithi13140796-20:head# 0x14000~27000 - have 0x14000 (81920) bytes fadvise_flags 0x0 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x14000~27000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8417b800 20 blobs 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 fbmap_alloc 0x558e80233300 allocate 0x28000/1000,28000,0 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x10d000~a000/1000,28000,0 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x156000~1e000/1000,28000,0 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x10d000~a000,0x156000~1e000] 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194e5b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194fa40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194e5b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194fa40 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e8194e5b0) get_ref 0x0~2000 Blob(0x558e8194e5b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194fa40 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e8194e5b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e8194e5b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fa40 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8194fa58 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194f880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e9490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194f880 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9490 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e8194f880) get_ref 0x0~2000 Blob(0x558e8194f880 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9490 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e8194f880) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e8194f880 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9490 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e817e94a8 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81866bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a2f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81866bd0 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a2f50 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e81866bd0) get_ref 0x0~2000 Blob(0x558e81866bd0 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a2f50 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e81866bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e81866bd0 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2f50 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a2f68 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a2fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a3030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a2fc0 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3030 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a2fc0) get_ref 0x0~2000 Blob(0x558e841a2fc0 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3030 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a2fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e841a2fc0 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3030 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a3048 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 fbmap_alloc 0x558e80233300 allocate 0x27000/1000,27000,0 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x174000~4000/1000,27000,0 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x273000~23000/1000,27000,0 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a30a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a3110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x174000~4000,0x273000~23000] 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a30a0 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3110 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f95260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f951f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a30a0) get_ref 0x0~2000 Blob(0x558e841a30a0 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3110 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a30a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f951f0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e841a30a0 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3110 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a3128 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95260) get_ref 0x0~2000 Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f951f0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95260) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f951f0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83f95208 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a3180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a31f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f95180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f950a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a3180 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a31f0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f950a0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3180) get_ref 0x0~2000 Blob(0x558e841a3180 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a31f0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3180) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95180) get_ref 0x0~2000 Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f950a0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95180) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e841a3180 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a31f0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a3208 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f950a0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83f950b8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a3260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a32d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f95030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edecb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a3260 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a32d0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edecb0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3260) get_ref 0x0~2000 Blob(0x558e841a3260 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a32d0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3260) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e841a3260 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a32d0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a32e8 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a3340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a33b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a3340 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a33b0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95030) get_ref 0x0~2000 Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edecb0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3340) get_ref 0x0~2000 Blob(0x558e841a3340 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a33b0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83f95030) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3340) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e841a3340 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a33b0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edecb0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a33c8 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83edecc8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a3420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a3490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83edef50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d0bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a3420 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3490 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d0bd0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3420) get_ref 0x0~2000 Blob(0x558e841a3420 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3490 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3420) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83edef50) get_ref 0x0~2000 Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d0bd0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e841a3420 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3490 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e83edef50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a34a8 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0bd0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e811d0be8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a3500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a3570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a3500 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3570 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e811d0a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d0850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3500) get_ref 0x0~2000 Blob(0x558e841a3500 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3570 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3500) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d0850 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e841a3500 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3570 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a3588 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e811d0a80) get_ref 0x0~2000 Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d0850 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e811d0a80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0850 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e811d0868 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a35e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a3650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a35e0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3650 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84014310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a35e0) get_ref 0x0~2000 Blob(0x558e841a35e0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3650 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a35e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5b90 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e841a35e0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3650 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a3668 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e84014310) get_ref 0x0~2000 Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5b90 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e84014310) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5b90 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a36c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a3730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5ba8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a36c0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3730 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a36c0) get_ref 0x0~2000 Blob(0x558e841a36c0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3730 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a36c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e8cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e89a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e841a36c0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3730 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e89a0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a3748 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e817e8cb0) get_ref 0x0~2000 Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e89a0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e817e8cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a37a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a3810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a37a0 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3810 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e89a0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e817e89b8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a37a0) get_ref 0x0~2000 Blob(0x558e841a37a0 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3810 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a37a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e841a37a0 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3810 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81867180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e80f04150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a3828 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e80f04150 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a3880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a38f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81867180) get_ref 0x0~2000 Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e80f04150 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a3880 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a38f0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81867180) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3880) get_ref 0x0~2000 Blob(0x558e841a3880 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a38f0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3880) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e80f04150 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e80f04168 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e841a3880 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a38f0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a3908 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a90000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a90070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a3960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a39d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90070 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a3960 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a39d0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a90000) get_ref 0x0~2000 Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90070 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a90000) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3960) get_ref 0x0~2000 Blob(0x558e841a3960 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a39d0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3960) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90070 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e81a90088 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558e841a3960 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a39d0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a39e8 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a900e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a90150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a3a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a3ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90150 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a3a40 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3ab0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a900e0) get_ref 0x0~2000 Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90150 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a900e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3a40) get_ref 0x0~2000 Blob(0x558e841a3a40 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3ab0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3a40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90150 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558e841a3a40 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3ab0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e81a90168 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a3ac8 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a901c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a90230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90230 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a3b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a3b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a901c0) get_ref 0x0~2000 Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90230 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a901c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a3b20 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3b90 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90230 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e81a90248 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3b20) get_ref 0x0~2000 Blob(0x558e841a3b20 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3b90 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3b20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558e841a3b20 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3b90 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a902a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a90310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a3ba8 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90310 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a902a0) get_ref 0x0~2000 Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90310 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a3c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a3c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a902a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a3c00 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3c70 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90310 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3c00) get_ref 0x0~2000 Blob(0x558e841a3c00 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3c70 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3c00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e81a90328 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558e841a3c00 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3c70 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a90380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a903f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a3c88 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a903f0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a90380) get_ref 0x0~2000 Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a903f0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a90380) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a3ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a3d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a903f0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a3ce0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3d50 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e81a90408 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3ce0) get_ref 0x0~2000 Blob(0x558e841a3ce0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3d50 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a90460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a904d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558e841a3ce0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3d50 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a3d68 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a904d0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a90460) get_ref 0x0~2000 Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a904d0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a3dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a3e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a90460) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a3dc0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3e30 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a904d0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e81a904e8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3dc0) get_ref 0x0~2000 Blob(0x558e841a3dc0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a3e30 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.blob(0x558e841a3dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a90540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a905b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558e841a3dc0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3e30 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e841a3e48 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a905b0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x36000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a90540) get_ref 0x0~2000 Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a905b0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a90540) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dirty_range mark inline shard dirty 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a905b0 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e81a905c8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xe000~28000 = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a90620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a90690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90690 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a90620) get_ref 0x0~2000 Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90690 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a90620) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90690 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e81a906a8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a90700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a90770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90770 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.198F0B8C.4.smithi13140796-14..' <- OBJ_.1_0000000000000001.198F0B8C.4.smithi13140796-14.. 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a90700) get_ref 0x0~2000 Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90770 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a90700) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90770 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e81a90788 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.1_0000000000000001.198F0B8C.4.smithi13140796-14..' <- SNA_1_0000000000000004_.1_0000000000000001.198F0B8C.4.smithi13140796-14.. 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a907e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a90850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90850 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a907e0) get_ref 0x0~2000 Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90850 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a907e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90850 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.0000000027.00000000000000000004' <- 0000000027.00000000000000000004 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e81a90868 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.0000000027.00000000000000000005' <- 0000000027.00000000000000000005 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a908c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a90930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90930 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c000 cost 252283 (21 ios * 4000 + 168283 bytes) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a908c0) get_ref 0x0~2000 Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90930 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a908c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c000 onodes 0x558e811dd680,0x558e8125c280,0x558e8417ca00 shared_blobs 0x558e817e9650,0x558e817e96c0,0x558e817e9b90,0x558e817e9c70,0x558e83c64070,0x558e83c657a0,0x558e83c658f0 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90930 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e81a90948 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) update 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a909a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a90a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) update inline shard 572 bytes from 27 extents 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a909a0 blob([0x295000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90a10 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a909a0) get_ref 0x0~1000 Blob(0x558e81a909a0 blob([0x295000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a90a10 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.blob(0x558e81a909a0) get_ref init 0x1000, 1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3a000~1000: 0x0~1000 Blob(0x558e81a909a0 blob([0x295000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e81a90a10 sbid 0x0)) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e81a90a28 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x3b000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) dirty_range mark inline shard dirty 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x14000~27000 = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:891f0d13:::smithi13140796-14:head# is 55040 (54462 bytes onode + 2 bytes spanning blobs + 576 bytes inline extents) 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.FE9893EA.4.smithi13140796-20..' <- OBJ_.1_0000000000000001.FE9893EA.4.smithi13140796-20.. 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.1_0000000000000001.FE9893EA.4.smithi13140796-20..' <- SNA_1_0000000000000004_.1_0000000000000001.FE9893EA.4.smithi13140796-20.. 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.624+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.0000000027.00000000000000000007' <- 0000000027.00000000000000000007 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.0000000027.00000000000000000008' <- 0000000027.00000000000000000008 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e8125c388) update #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c0371625:::snapmapper:0# is 29 (21 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e8417cb08) update 1#1:891f0d13:::smithi13140796-14:4# 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e8417cb08) update inline shard 192 bytes from 7 extents 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417b800 cost 248187 (21 ios * 4000 + 164187 bytes) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417b800 onodes 0x558e83c72280,0x558e83c72500 shared_blobs 0x558e8194e4d0,0x558e8194e8c0,0x558e8194f810,0x558e83af92d0,0x558e83af93b0,0x558e83af9490,0x558e83af9570,0x558e83c65ab0,0x558e83c65b20,0x558e83c65c00 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72388) update 1#1:f7919c75:::smithi13140796-20:4# 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:891f0d13:::smithi13140796-14:4# is 54515 (54317 bytes onode + 2 bytes spanning blobs + 196 bytes inline extents) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72388) update inline shard 273 bytes from 10 extents 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:f7919c75:::smithi13140796-20:4# is 691 (412 bytes onode + 2 bytes spanning blobs + 277 bytes inline extents) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) update 1#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) update inline shard 650 bytes from 30 extents 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:f7919c75:::smithi13140796-20:head# is 1213 (557 bytes onode + 2 bytes spanning blobs + 654 bytes inline extents) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2804 is 11 SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2808 is 11 SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2806 is 11 SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x280e is 11 SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x280f is 11 SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2810 is 11 SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2811 is 11 SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x280c is 11 SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x280a is 11 SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x280d is 11 SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417b800 allocated 0x[174000~4000,273000~23000] released 0x[] 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 freelist allocate 0x174000~4000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 freelist allocate 0x273000~23000 2019-12-07T19:34:04.624+0000 7fe2c425f700 20 freelist _xor first_key 0x200000 last_key 0x280000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2807 is 11 SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2809 is 11 SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2805 is 11 SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x280b is 11 SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 prepare 2019-12-07T19:34:04.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8417b800 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2801 is 11 SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2802 is 11 SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2803 is 11 SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2))) 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c000 allocated 0x[10d000~a000,156000~1e000] released 0x[] 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 freelist allocate 0x10d000~a000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 freelist allocate 0x156000~1e000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 20 freelist _xor first_key 0x100000 last_key 0x100000 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 prepare 2019-12-07T19:34:04.624+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83d4c000 2019-12-07T19:34:04.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:04.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.624+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x88589~416 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.624+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x88589 0x88589~416 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.624+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.624+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x58589 2019-12-07T19:34:04.624+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x589 2019-12-07T19:34:04.624+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.624+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x99f and padding block with 0x661 2019-12-07T19:34:04.625+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 76 ==== MRecoveryReserve(1.3s5 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e84199860 con 0x558e818fe400 2019-12-07T19:34:04.625+0000 7fe2e6d3a700 15 osd.0 27 enqueue_peering_evt 1.3s5 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.625+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.625+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 77 ==== MRecoveryReserve(1.bs1 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e84199860 con 0x558e818fe400 2019-12-07T19:34:04.625+0000 7fe2e6d3a700 15 osd.0 27 enqueue_peering_evt 1.bs1 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.625+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.625+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 74 ==== pg_info2(1.2s0 1.2s0 1.2s5( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558e8404d800 con 0x558e818d7000 2019-12-07T19:34:04.625+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(5) info: 1.2s5( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s) 2019-12-07T19:34:04.625+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(5) info: 1.2s5( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.625+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.625+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.625+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.625+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558e81231000 2019-12-07T19:34:04.625+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.625+0000 7fe2ca26b700 5 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.626489+0000 3 0.000200 2019-12-07T19:34:04.625+0000 7fe2ca26b700 5 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.625+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:04.625+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.625+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.625+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.625+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558e83c78000 2019-12-07T19:34:04.625+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.625+0000 7fe2ca26b700 5 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.626545+0000 3 0.000205 2019-12-07T19:34:04.625+0000 7fe2ca26b700 5 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.625+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:04.625+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] op_applied version 27'8 2019-12-07T19:34:04.626+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e83b0be40 finish 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=99, to_read={1:ec1285ab:::smithi13140796-19:head=0,77824,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=99, to_read={1:ec1285ab:::smithi13140796-19:head=0,77824,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) pg 0x558e8122e000 2019-12-07T19:34:04.626+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e84176420 prio 127 cost 0 latency 0.002863 MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=99, to_read={1:ec1285ab:::smithi13140796-19:head=0,77824,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.626+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=99, to_read={1:ec1285ab:::smithi13140796-19:head=0,77824,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.626+0000 7fe2c425f700 25 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.626+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 0x0~13000 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~13000 size 0x13000 (77824) 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8e70 blob([0x8b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8150 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af80e0 blob([0x8d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8070 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5ab0 blob([0x8f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8f50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e90a0 blob([0x91000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af81c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8230 blob([0x93000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af82a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8310 blob([0x95000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8380 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af83f0 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8460 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af84d0 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8540 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af85b0 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8620 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8690 blob([0x9d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83af8700 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e8e70 blob([0x8b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8150 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e90a0 blob([0x91000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af81c0 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac5ab0 blob([0x8f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8f50 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af80e0 blob([0x8d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8070 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af8230 blob([0x93000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af82a0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af8310 blob([0x95000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8380 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af83f0 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8460 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af84d0 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8540 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af85b0 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8620 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af8690 blob([0x9d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83af8700 sbid 0x0)) need {<0x0, 0x1000> : [0x12000:0~1000]} 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.626+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.626+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.626+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'5 (0'0,27'5] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'3 crt=21'3 lcod 21'3 active+remapped mbc={} ps=[2~1]] op_applied version 27'5 2019-12-07T19:34:04.626+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e83b0b600 finish 2019-12-07T19:34:04.626+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 131 ==== MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=101, reqid=client.4540.0:189, at_version=27'16, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2 ==== 1860+0+0 (unknown 0 0 0) 0x558e83acb000 con 0x558e818d7c00 2019-12-07T19:34:04.626+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.626+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84176580 prio 127 cost 0 latency 0.000040 epoch 27 MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=101, reqid=client.4540.0:189, at_version=27'16, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2 2019-12-07T19:34:04.626+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=101, reqid=client.4540.0:189, at_version=27'16, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.626+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x8899f 2019-12-07T19:34:04.626+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.626+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.626+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.626+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.626+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.626+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(5) info: 1.2s5( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.626+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.626+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(5) info: 1.2s5( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s)) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.626+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 132 ==== pg_info2(1.as3 1.as3 1.as0( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558e8404d800 con 0x558e818d7c00 2019-12-07T19:34:04.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.001961654s (0.000000201s flush + 0.001961453s kv commit) 2019-12-07T19:34:04.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.626+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.as3 epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.066931508s ub 44.066931508s int 16s) 2019-12-07T19:34:04.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.626+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.066931508s ub 44.066931508s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0a900> 2019-12-07T19:34:04.626+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 3(5) info: 1.2s5( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s) 2019-12-07T19:34:04.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 kv_submitted 2019-12-07T19:34:04.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a900 2019-12-07T19:34:04.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 finishing 2019-12-07T19:34:04.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a900 onodes 2019-12-07T19:34:04.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a900 done 2019-12-07T19:34:04.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a900 [] 2019-12-07T19:34:04.626+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.626+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recalc_readable_until peer osd.4 ruub 44.065221680s 2019-12-07T19:34:04.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.626+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:04.626+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.626+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=101, reqid=client.4540.0:189, at_version=27'16, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.626+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.626+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recalc_readable_until peer osd.3 ruub 44.065221680s 2019-12-07T19:34:04.626+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=101, reqid=client.4540.0:189, at_version=27'16, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) pg 0x558e841a4000 2019-12-07T19:34:04.626+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.626+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recalc_readable_until readable_until[_ub] 0s (sent 44.065221680s) 2019-12-07T19:34:04.626+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e84176580 prio 127 cost 0 latency 0.000142 MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=101, reqid=client.4540.0:189, at_version=27'16, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2 pg pg[1.8s5( v 23'14 lc 21'3 (0'0,23'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'14 lcod 0'0 active m=6 mbc={} ps=[2~1]] 2019-12-07T19:34:04.626+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] state: peer osd.3(5) activated and committed 2019-12-07T19:34:04.626+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 23'14 lc 21'3 (0'0,23'14] local-lis/les=0/0 n=5 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'14 lcod 0'0 active m=6 mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=101, reqid=client.4540.0:189, at_version=27'16, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2 2019-12-07T19:34:04.626+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.626+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.626+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.626+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 23'14 lc 21'3 (0'0,23'14] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'14 lcod 0'0 active m=7 mbc={} ps=[2~1]] append_log log((0'0,23'14], crt=23'14) [27'15 (21'8) clone 1:10779340:::smithi13140796-32:4 by unknown.0.0:0 2019-12-07T19:33:58.703940+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,27'16 (21'8) modify 1:10779340:::smithi13140796-32:head by client.4540.0:189 2019-12-07T19:34:01.651210+0000 0 ObjectCleanRegions clean_offsets: [0~524288,1146880~18446744073708404735], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.626+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.626+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.626+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'15 lc 21'3 (0'0,27'15] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'14 crt=27'15 lcod 0'0 active m=7 mbc={} ps=[2~1]] add_log_entry 27'15 (21'8) clone 1:10779340:::smithi13140796-32:4 by unknown.0.0:0 2019-12-07T19:33:58.703940+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.627+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'16 lc 21'3 (0'0,27'16] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'14 crt=27'16 lcod 0'0 active m=7 mbc={} ps=[2~1]] add_log_entry 27'16 (21'8) modify 1:10779340:::smithi13140796-32:head by client.4540.0:189 2019-12-07T19:34:01.651210+0000 0 ObjectCleanRegions clean_offsets: [0~524288,1146880~18446744073708404735], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.627+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'16 lc 21'3 (0'0,27'16] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'14 crt=27'16 lcod 0'0 active m=7 mbc={} ps=[2~1]] append_log approx pg log length = 16 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.627+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'16 lc 21'3 (0'0,27'16] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'14 crt=27'16 lcod 0'0 active m=7 mbc={} ps=[2~1]] append_log transaction_applied = 0 2019-12-07T19:34:04.627+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'16 lc 21'3 (0'0,27'16] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'14 crt=27'16 lcod 0'0 active m=7 mbc={} ps=[2~1]] append_log 0(5) is async_recovery or backfill target 2019-12-07T19:34:04.627+0000 7fe2cba6e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.627+0000 7fe2cba6e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'15, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(4)=3} 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 4 2019-12-07T19:34:04.627+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 4 2019-12-07T19:34:04.627+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e811faf00 seq 4 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 8 2019-12-07T19:34:04.627+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.627+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000027.00000000000000000015' <- 0000000027.00000000000000000015 2019-12-07T19:34:04.627+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000027.00000000000000000016' <- 0000000027.00000000000000000016 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.627+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.627+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.627+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.missing/0000000000000001.80EE9C20.4.smithi13140796-32..' <- missing/0000000000000001.80EE9C20.4.smithi13140796-32.. 2019-12-07T19:34:04.627+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.missing/0000000000000001.80EE9C20.head.smithi13140796-32..' <- missing/0000000000000001.80EE9C20.head.smithi13140796-32.. 2019-12-07T19:34:04.627+0000 7fe2c6a64700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 27:44 2019-12-07T19:34:04.627+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.627+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.627+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.627+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811faf00 cost 6662 (1 ios * 4000 + 2662 bytes) 2019-12-07T19:34:04.627+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811faf00 onodes shared_blobs 2019-12-07T19:34:04.627+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811faf00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.066931508s ub 44.066931508s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.066931508s ub 44.066931508s int 16s)) prio 255 cost 10 e27) pg 0x558e8193b000 2019-12-07T19:34:04.627+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 prepare 2019-12-07T19:34:04.627+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811faf00 2019-12-07T19:34:04.627+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 io_done 2019-12-07T19:34:04.627+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.066931508s ub 44.066931508s int 16s) 2019-12-07T19:34:04.627+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'16 lc 21'3 (0'0,27'16] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'14 crt=27'16 lcod 0'0 active m=7 mbc={} ps=[2~1]] op_applied version 27'16 2019-12-07T19:34:04.627+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e84176580 finish 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.627+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.627+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.627+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.627+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e81a0bb00 seq 15 2019-12-07T19:34:04.627+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2499 < min_flush_size 524288 2019-12-07T19:34:04.627+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:04.627+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0bb00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0bb00 onodes shared_blobs 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.627+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 prepare 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0bb00 2019-12-07T19:34:04.627+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 io_done 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.627+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2522 < min_flush_size 524288 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2522 < min_flush_size 524288 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x8899f~9da to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x8899f 0x8899f~9da to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x5899f 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x99f 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x379 and padding block with 0xc87 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x89379 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000112606s (0.000000078s flush + 0.000112528s kv commit) 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.627+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 aio_wait 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x89379~416 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.627+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417b800 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x89379 0x89379~416 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~60000 x_off 0x59379 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x379 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x78f and padding block with 0x871 2019-12-07T19:34:04.627+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 io_done 2019-12-07T19:34:04.627+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 aio_wait 2019-12-07T19:34:04.627+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c000 2019-12-07T19:34:04.627+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 io_done 2019-12-07T19:34:04.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811faf00> 2019-12-07T19:34:04.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.627+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 kv_submitted 2019-12-07T19:34:04.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811faf00 2019-12-07T19:34:04.627+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 finishing 2019-12-07T19:34:04.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811faf00 onodes 2019-12-07T19:34:04.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811faf00 done 2019-12-07T19:34:04.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.627+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811faf00 [] 2019-12-07T19:34:04.627+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.627+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.627+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 27/26 ECSubWriteReply(tid=101, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e84197600 con 0x558e818d7c00 2019-12-07T19:34:04.627+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e8e70 blob([0x8b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8150 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8168 in 0x558e80219880) _discard 0x1000~1000 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e811707e0 space 0x558e83af8168 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e90a0 blob([0x91000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af81c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af81d8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e83c8e360 space 0x558e83af81d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac5ab0 blob([0x8f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8f50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e817e8f68 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84033e60 space 0x558e817e8f68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af80e0 blob([0x8d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8070 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8088 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84033ef0 space 0x558e83af8088 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af8230 blob([0x93000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af82a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af82b8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e841ec480 space 0x558e83af82b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af8310 blob([0x95000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8380 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8398 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e841ec510 space 0x558e83af8398 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af83f0 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8460 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8478 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e841ec5a0 space 0x558e83af8478 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af84d0 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8540 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8558 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e841ec630 space 0x558e83af8558 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af85b0 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8620 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8638 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e841ec6c0 space 0x558e83af8638 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af8690 blob([0x9d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83af8700 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12000:0~1000]} 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8718 in 0x558e80219880) _discard 0x0~1000 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e841ec750 space 0x558e83af8718 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.627+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 0x0~13000 = 77824 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=77824 r=77824 len=77824 2019-12-07T19:34:04.627+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:04.627+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:04.627+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:04.627+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 77824 2019-12-07T19:34:04.627+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# hinfo_key 2019-12-07T19:34:04.627+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# hinfo_key = 0 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:04.627+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 27/26 ECSubReadReply(tid=99, attrs_read=0)) v2 -- 0x558e83c73900 con 0x558e818d7c00 2019-12-07T19:34:04.627+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e84176420 finish 2019-12-07T19:34:04.627+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x8978f 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000292831s (0.000000059s flush + 0.000292772s kv commit) 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=2 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4883 < min_flush_size 524288 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72280 had 1 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72500 had 1 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 2 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 26737 < min_flush_size 524288 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 59505 < min_flush_size 524288 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 92273 < min_flush_size 524288 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 117450 < min_flush_size 524288 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dd680 had 1 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417ca00 had 1 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 117473 < min_flush_size 524288 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 117473 < min_flush_size 524288 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x8978f~1cae1 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x8978f 0x8978f~1cae1 to file(ino 16 size 0x90000 mtime 2019-12-07T19:34:04.612611+0000 allocated 90000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~60000]) 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _allocate len 0x16270 from 0 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _allocate len 0x16270 from 1 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x20000/10000,20000,acd1c0000 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd1c0000~20000/10000,20000,acd1c0000 2019-12-07T19:34:04.627+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0xb0000 to include allocated 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 31 (was clean) 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~80000 x_off 0x5978f 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x78f 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x270 and padding block with 0xd90 2019-12-07T19:34:04.627+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0xb0000 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0bb00> 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.628+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 kv_submitted 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0bb00 2019-12-07T19:34:04.628+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 finishing 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0bb00 onodes 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0bb00 done 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:04.628+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0bb00 [] 2019-12-07T19:34:04.628+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.628+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.628+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.628+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xa6270 2019-12-07T19:34:04.628+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.628+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.628+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.628+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.628+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (31) on file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]), flushing log 2019-12-07T19:34:04.628+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:04.628+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.628+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 31 len 0x49 crc 0x7038c8d4) 2019-12-07T19:34:04.628+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x1e000~1000 to file(ino 1 size 0x1e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.628+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x1e000 0x1e000~1000 to file(ino 1 size 0x1e000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.628+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x1f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.628+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1e000 2019-12-07T19:34:04.628+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x1f000 2019-12-07T19:34:04.628+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:04.628+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:34:04.628+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000025 2019-12-07T19:34:04.628+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.628+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 31 2019-12-07T19:34:04.628+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.628+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000695804s (0.000003369s flush + 0.000692435s kv commit) 2019-12-07T19:34:04.628+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417b800,0x558e83d4c000> 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.628+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 kv_submitted 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417b800 2019-12-07T19:34:04.628+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 finishing 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417b800 onodes 0x558e83c72280,0x558e83c72500 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e80f04168 in 0x558e8102e000) _finish_write discard buffer(0x558e81171ef0 space 0x558e80f04168 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d0868 in 0x558e8102e000) _finish_write discard buffer(0x558e81170630 space 0x558e811d0868 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d0be8 in 0x558e8102e000) _finish_write discard buffer(0x558e811706c0 space 0x558e811d0be8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e89b8 in 0x558e8102e000) _finish_write discard buffer(0x558e81170fc0 space 0x558e817e89b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a90088 in 0x558e8102e000) _finish_write discard buffer(0x558e83d119e0 space 0x558e81a90088 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a90168 in 0x558e8102e000) _finish_write discard buffer(0x558e83d102d0 space 0x558e81a90168 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a90248 in 0x558e8102e000) _finish_write discard buffer(0x558e83d10240 space 0x558e81a90248 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a90328 in 0x558e8102e000) _finish_write discard buffer(0x558e84033560 space 0x558e81a90328 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a90408 in 0x558e8102e000) _finish_write discard buffer(0x558e84033680 space 0x558e81a90408 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a904e8 in 0x558e8102e000) _finish_write discard buffer(0x558e84033710 space 0x558e81a904e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a905c8 in 0x558e8102e000) _finish_write discard buffer(0x558e840335f0 space 0x558e81a905c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a906a8 in 0x558e8102e000) _finish_write discard buffer(0x558e840337a0 space 0x558e81a906a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a90788 in 0x558e8102e000) _finish_write discard buffer(0x558e84032a20 space 0x558e81a90788 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a90868 in 0x558e8102e000) _finish_write discard buffer(0x558e84033a70 space 0x558e81a90868 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a90948 in 0x558e8102e000) _finish_write discard buffer(0x558e84033950 space 0x558e81a90948 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a90a28 in 0x558e8102e000) _finish_write discard buffer(0x558e84032240 space 0x558e81a90a28 0x0~1000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac5ba8 in 0x558e8102e000) _finish_write discard buffer(0x558e81171cb0 space 0x558e83ac5ba8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edecc8 in 0x558e8102e000) _finish_write discard buffer(0x558e811703f0 space 0x558e83edecc8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f950b8 in 0x558e8102e000) _finish_write discard buffer(0x558e81170ab0 space 0x558e83f950b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f95208 in 0x558e8102e000) _finish_write discard buffer(0x558e811718c0 space 0x558e83f95208 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b800 done 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:04.628+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417b800 [] 2019-12-07T19:34:04.628+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.628+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 kv_submitted 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c000 2019-12-07T19:34:04.628+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 finishing 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c000 onodes 0x558e811dd680,0x558e8125c280,0x558e8417ca00 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e94a8 in 0x558e80219500) _finish_write discard buffer(0x558e81170000 space 0x558e817e94a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194fa58 in 0x558e80219500) _finish_write discard buffer(0x558e83c8f050 space 0x558e8194fa58 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a2f68 in 0x558e80219500) _finish_write discard buffer(0x558e840330e0 space 0x558e841a2f68 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a3048 in 0x558e80219500) _finish_write discard buffer(0x558e84033170 space 0x558e841a3048 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a3128 in 0x558e80219500) _finish_write discard buffer(0x558e81171e60 space 0x558e841a3128 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a3208 in 0x558e80219500) _finish_write discard buffer(0x558e81170900 space 0x558e841a3208 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a32e8 in 0x558e80219500) _finish_write discard buffer(0x558e81170cf0 space 0x558e841a32e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a33c8 in 0x558e80219500) _finish_write discard buffer(0x558e81171b00 space 0x558e841a33c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a34a8 in 0x558e80219500) _finish_write discard buffer(0x558e81170c60 space 0x558e841a34a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a3588 in 0x558e80219500) _finish_write discard buffer(0x558e81170360 space 0x558e841a3588 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a3668 in 0x558e80219500) _finish_write discard buffer(0x558e84033320 space 0x558e841a3668 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a3748 in 0x558e80219500) _finish_write discard buffer(0x558e840333b0 space 0x558e841a3748 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a3828 in 0x558e80219500) _finish_write discard buffer(0x558e84033440 space 0x558e841a3828 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a3908 in 0x558e80219500) _finish_write discard buffer(0x558e84033200 space 0x558e841a3908 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a39e8 in 0x558e80219500) _finish_write discard buffer(0x558e84033290 space 0x558e841a39e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a3ac8 in 0x558e80219500) _finish_write discard buffer(0x558e840334d0 space 0x558e841a3ac8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a3ba8 in 0x558e80219500) _finish_write discard buffer(0x558e81171050 space 0x558e841a3ba8 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a3c88 in 0x558e80219500) _finish_write discard buffer(0x558e841ec000 space 0x558e841a3c88 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a3d68 in 0x558e80219500) _finish_write discard buffer(0x558e841ec090 space 0x558e841a3d68 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a3e48 in 0x558e80219500) _finish_write discard buffer(0x558e841ec120 space 0x558e841a3e48 0x0~2000 writing nocache) 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c000 done 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:04.628+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c000 [] 2019-12-07T19:34:04.628+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.628+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.628+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 27/26 ECSubWriteReply(tid=49, last_complete=27'8, committed=1, applied=1)) v2 -- 0x558e80264000 con 0x558e818d7000 2019-12-07T19:34:04.628+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.628+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 27/26 ECSubWriteReply(tid=50, last_complete=27'5, committed=1, applied=1)) v2 -- 0x558e818a8c00 con 0x558e818d7000 2019-12-07T19:34:04.628+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.628+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.628+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.629+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 133 ==== pg_info2(1.2s0 1.2s0 1.2s2( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1044+0+0 (unknown 0 0 0) 0x558e8404d800 con 0x558e818d7c00 2019-12-07T19:34:04.629+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(2) info: 1.2s2( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s) 2019-12-07T19:34:04.629+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(2) info: 1.2s2( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(2) info: 1.2s2( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(2) info: 1.2s2( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s)) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.629+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 1(2) info: 1.2s2( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease_ack(ruub 44.065221680s) 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recalc_readable_until peer osd.4 ruub 44.065221680s 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recalc_readable_until peer osd.1 ruub 44.065221680s 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recalc_readable_until peer osd.3 ruub 44.065221680s 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recalc_readable_until readable_until[_ub] 44.065221680s (sent 44.065221680s) 2019-12-07T19:34:04.629+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] state: peer osd.1(2) activated and committed 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(4)=3} 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 4 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 4 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 8 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.629+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 27:45 2019-12-07T19:34:04.629+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] state: all_activated_and_committed 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(4)=3} 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 4 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 4 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 8 2019-12-07T19:34:04.629+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 activating+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.629+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] state: mnow 28.085348501s >= prior_readable_until_ub 0s 2019-12-07T19:34:04.629+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] share_pg_info 2019-12-07T19:34:04.630+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e8122b100 con 0x558e818d7c00 2019-12-07T19:34:04.630+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e81229500 con 0x558e818d7000 2019-12-07T19:34:04.630+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e80ff2300 con 0x558e818fe400 2019-12-07T19:34:04.630+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e80ff3800 con 0x558e818d6800 2019-12-07T19:34:04.630+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_info2(1.2s4 1.2s4 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 -- 0x558e83c9f100 con 0x558e818fe000 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(4)=3} 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 4 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 4 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 8 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.630+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 27:46 2019-12-07T19:34:04.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] check_local 2019-12-07T19:34:04.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] checking 1:4c9e444c:::smithi13140796-48:head at 23'10 2019-12-07T19:34:04.630+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] requeue_op 0x558e819438c0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e27) 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] requeue_op 0x558e81943080 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e27) 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] requeue_op 0x558e819431e0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e27) 2019-12-07T19:34:04.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] needs_recovery osd.5(3) has 4 missing 2019-12-07T19:34:04.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] activate not all replicas are up-to-date, queueing recovery 2019-12-07T19:34:04.630+0000 7fe2caa6c700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 DoRecovery 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 DoRecovery) prio 255 cost 10 e27) 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(4)=3} 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 4 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 4 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 8 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.630+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 27:47 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] hit_set_clear 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] agent_stop 2019-12-07T19:34:04.630+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.630+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e8195f500 seq 26 2019-12-07T19:34:04.630+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:04.630+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195f500 cost 5310 (1 ios * 4000 + 1310 bytes) 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195f500 onodes shared_blobs 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195f500 allocated 0x[] released 0x[] 2019-12-07T19:34:04.630+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 prepare 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195f500 2019-12-07T19:34:04.630+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 io_done 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 DoRecovery) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 DoRecovery) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.630+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 DoRecovery 2019-12-07T19:34:04.630+0000 7fe2caa6c700 5 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] exit Started/Primary/Active/Activating 2019-12-07T19:34:04.631470+0000 14 0.004639 2019-12-07T19:34:04.630+0000 7fe2caa6c700 5 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] enter Started/Primary/Active/WaitLocalRecoveryReserved 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_recovery_priority recovery priority is 190 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(4)=3} 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.630+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 4 2019-12-07T19:34:04.631+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 4 2019-12-07T19:34:04.631+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 8 2019-12-07T19:34:04.631+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.631+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.631+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 27:48 2019-12-07T19:34:04.631+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.631+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.631+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 134 ==== MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=102, reqid=client.4540.0:190, at_version=27'17, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2 ==== 1724+0+0 (unknown 0 0 0) 0x558e83e55000 con 0x558e818d7c00 2019-12-07T19:34:04.631+0000 7fe2cea74700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 LocalRecoveryReserved 2019-12-07T19:34:04.631+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e27) queued 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.631+0000 7fe2cea74700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 LocalRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:04.631+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e841766e0 prio 127 cost 0 latency 0.000033 epoch 27 MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=102, reqid=client.4540.0:190, at_version=27'17, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2 2019-12-07T19:34:04.631+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=102, reqid=client.4540.0:190, at_version=27'17, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1164 < min_flush_size 524288 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1164 < min_flush_size 524288 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xa6270~48c to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xa6270 0xa6270~48c to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~80000 x_off 0x76270 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x270 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x6fc and padding block with 0x904 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 LocalRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e27) pg 0x558e81075000 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=102, reqid=client.4540.0:190, at_version=27'17, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e819431e0 prio 63 cost 4 latency 2.977565 osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=102, reqid=client.4540.0:190, at_version=27'17, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) pg 0x558e841a4000 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 27 send_incremental_map 26 -> 27 to 0x558e83d68000 172.21.15.131:0/224485236 2019-12-07T19:34:04.631+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_map(27..27 src has 1..27) v4 -- 0x558e81222900 con 0x558e83d68000 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e841766e0 prio 127 cost 0 latency 0.000165 MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=102, reqid=client.4540.0:190, at_version=27'17, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2 pg pg[1.8s5( v 27'16 lc 21'3 (0'0,27'16] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active m=7 mbc={} ps=[2~1]] 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 27 maybe_share_map con 0x558e83d68000 172.21.15.131:0/224485236 map epoch 26 -> 27 (shared) 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'16 lc 21'3 (0'0,27'16] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active m=7 mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=102, reqid=client.4540.0:190, at_version=27'17, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: osd_op(client.4540.0:195 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'16 lc 21'3 (0'0,27'16] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'16 active m=7 mbc={} ps=[2~1]] append_log log((0'0,27'16], crt=27'16) [27'17 (27'16) modify 1:10779340:::smithi13140796-32:head by client.4540.0:190 2019-12-07T19:34:01.653781+0000 0 ObjectCleanRegions clean_offsets: [0~1146880,1523712~18446744073708027903], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_op: op osd_op(client.4540.0:195 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~753664,checksum in=4b,getxattrs] snapc 0=[] ondisk+read+known_if_redirected e24) v8 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'17 lc 21'3 (0'0,27'17] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'16 crt=27'17 active m=7 mbc={} ps=[2~1]] add_log_entry 27'17 (27'16) modify 1:10779340:::smithi13140796-32:head by client.4540.0:190 2019-12-07T19:34:01.653781+0000 0 ObjectCleanRegions clean_offsets: [0~1146880,1523712~18446744073708027903], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'17 lc 21'3 (0'0,27'17] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'16 crt=27'17 active m=7 mbc={} ps=[2~1]] append_log approx pg log length = 17 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'17 lc 21'3 (0'0,27'17] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'16 crt=27'17 active m=7 mbc={} ps=[2~1]] append_log transaction_applied = 0 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'17 lc 21'3 (0'0,27'17] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'16 crt=27'17 active m=7 mbc={} ps=[2~1]] append_log 0(5) is async_recovery or backfill target 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.631+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'17, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_op osd_op(client.4540.0:195 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~753664,checksum in=4b,getxattrs] snapc 0=[] ondisk+read+known_if_redirected e24) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] readable_with_acting: locs:0(0),1(2),3(5),4(1),5(4) 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_object_context: obc NOT found in cache: 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.631+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# _ 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e811fa600 seq 5 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# _ = 0 2019-12-07T19:34:04.631+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000027.00000000000000000017' <- 0000000027.00000000000000000017 2019-12-07T19:34:04.631+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# snapset 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._fastinfo' <- _fastinfo 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# snapset = 0 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.missing/0000000000000001.80EE9C20.head.smithi13140796-32..' <- missing/0000000000000001.80EE9C20.head.smithi13140796-32.. 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa600 cost 5415 (1 ios * 4000 + 1415 bytes) 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa600 onodes shared_blobs 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 prepare 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa600 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] populate_obc_watchers 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 io_done 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'17 lc 21'3 (0'0,27'17] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'16 crt=27'17 active m=7 mbc={} ps=[2~1]] op_applied version 27'17 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.631+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e841766e0 finish 2019-12-07T19:34:04.631+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattrs 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattrs 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# = 0 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_object_context: creating obc from disk: 0x558e8115b440 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_object_context: 0x558e8115b440 1:41fe8a11:::smithi13140796-45:head rwstate(none n=0 w=0) oi: 1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e83dcc900 snapset: 0=[]:{} 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] find_object_context 1:41fe8a11:::smithi13140796-45:head @head oi=1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] readable_with_acting: locs:0(0),1(2),3(5),4(1),5(4) 2019-12-07T19:34:04.631+0000 7fe2c6a64700 25 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_op oi 1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_op obc obc(1:41fe8a11:::smithi13140796-45:head rwstate(read n=1 w=0)) 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx 0x558e8404ae00 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx 1:41fe8a11:::smithi13140796-45:head [read 0~753664,checksum in=4b,getxattrs] ov 21'6 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op 1:41fe8a11:::smithi13140796-45:head [read 0~753664,checksum in=4b,getxattrs] 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op read 0~753664 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_read 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] async_read noted for 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op checksum in=4b 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_checksum 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_checksum: async_read noted for 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xa66fc 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 0(0) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000539810s (0.000000107s flush + 0.000539703s kv commit) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1239 < min_flush_size 524288 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 4(1) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1262 < min_flush_size 524288 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1262 < min_flush_size 524288 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xa66fc~4ee to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xa66fc 0xa66fc~4ee to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~80000 x_off 0x766fc 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x6fc 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xbea and padding block with 0x416 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] start_read_op: starting ReadOp(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_read_op: starting read ReadOp(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195f500> 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.631+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 kv_submitted 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195f500 2019-12-07T19:34:04.631+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 finishing 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195f500 onodes 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f500 done 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:04.631+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195f500 [] 2019-12-07T19:34:04.631+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 27 send_incremental_map 21 -> 27 to 0x558e802a5400 v2:172.21.15.131:6822/37079 2019-12-07T19:34:04.631+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- osd_map(22..27 src has 1..27) v4 -- 0x558e80238fc0 con 0x558e802a5400 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 27 maybe_share_map con 0x558e802a5400 v2:172.21.15.131:6822/37079 map epoch 21 -> 27 (shared) 2019-12-07T19:34:04.631+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402fb80 con 0x558e802a5400 2019-12-07T19:34:04.631+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.2s2 27/26 ECSubRead(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402f400 con 0x558e818d7c00 2019-12-07T19:34:04.631+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpRead(1.2s5 27/26 ECSubRead(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e84242280 con 0x558e818d7000 2019-12-07T19:34:04.631+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.2s1 27/26 ECSubRead(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e84242500 con 0x558e818fe400 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xa6bea 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.631+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000182403s (0.000000065s flush + 0.000182338s kv commit) 2019-12-07T19:34:04.631+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_read_op: started ReadOp(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, []))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=0(0),1(2),3(5),4(1)) 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa600> 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.631+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 kv_submitted 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa600 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e819431e0 finish 2019-12-07T19:34:04.631+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 finishing 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa600 onodes 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa600 done 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.631+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa600 [] 2019-12-07T19:34:04.631+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.631+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e27) queued 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 LocalRecoveryReserved) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.631+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 27/26 ECSubWriteReply(tid=102, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e84065400 con 0x558e818d7c00 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 LocalRecoveryReserved 2019-12-07T19:34:04.631+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.631+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.631+0000 7fe2c6a64700 5 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] exit Started/Primary/Active/WaitLocalRecoveryReserved 2019-12-07T19:34:04.632540+0000 1 0.000284 2019-12-07T19:34:04.631+0000 7fe2c6a64700 5 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] enter Started/Primary/Active/WaitRemoteRecoveryReserved 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_recovery_priority recovery priority is 190 2019-12-07T19:34:04.631+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.2s2 REQUEST prio: 190 e27/27) v3 -- 0x558e819b0780 con 0x558e818d7c00 2019-12-07T19:34:04.631+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.631+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e27) queued 2019-12-07T19:34:04.631+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.631+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e27) pg 0x558e81075000 2019-12-07T19:34:04.631+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558e8402fb80 con 0x558e802a5400 2019-12-07T19:34:04.631+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81943080 prio 63 cost 4 latency 2.978224 osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.631+0000 7fe2d427f700 15 osd.0 27 enqueue_op 0x558e83edd340 prio 127 cost 0 latency 0.000183 epoch 27 MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.631+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.631+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: osd_op(client.4540.0:196 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 2019-12-07T19:34:04.631+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_op: op osd_op(client.4540.0:196 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~753664,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e24) v8 2019-12-07T19:34:04.631+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== osd_map(22..27 src has 1..27) v4 ==== 0+0+0 (unknown 0 0 0) 0x558e80238fc0 con 0x558e802a5400 2019-12-07T19:34:04.631+0000 7fe2d4a80700 20 osd.0 27 OSD::ms_dispatch: osd_map(22..27 src has 1..27) v4 2019-12-07T19:34:04.631+0000 7fe2d4a80700 10 osd.0 27 do_waiters -- start 2019-12-07T19:34:04.631+0000 7fe2d4a80700 10 osd.0 27 do_waiters -- finish 2019-12-07T19:34:04.631+0000 7fe2d4a80700 20 osd.0 27 _dispatch 0x558e80238fc0 osd_map(22..27 src has 1..27) v4 2019-12-07T19:34:04.631+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:04.631+0000 7fe2d4a80700 3 osd.0 27 handle_osd_map epochs [22,27], i have 27, src has [1,27] 2019-12-07T19:34:04.631+0000 7fe2d4a80700 10 osd.0 27 no new maps here, dropping 2019-12-07T19:34:04.631+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_op osd_op(client.4540.0:196 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~753664,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e24) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:04.631+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] readable_with_acting: locs:0(0),1(2),3(5),4(1),5(4) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_object_context: found obc in cache: 0x558e8115b440 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_object_context: 0x558e8115b440 1:41fe8a11:::smithi13140796-45:head rwstate(read n=1 w=0) oi: 1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e83dcc900 snapset: 0=[]:{} 2019-12-07T19:34:04.632+0000 7fe2e2ca2700 10 osd.0 27 tick_without_osd_lock 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] find_object_context 1:41fe8a11:::smithi13140796-45:head @head oi=1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] readable_with_acting: locs:0(0),1(2),3(5),4(1),5(4) 2019-12-07T19:34:04.632+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fd05000/0x40000000/0x1680000000, data 0x2ee94f/0x2f9000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 25 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_op oi 1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:34:04.632+0000 7fe2e2ca2700 25 osd.0 27 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254252+0000 last_rx_front 2019-12-07T19:34:02.254310+0000 2019-12-07T19:34:04.632+0000 7fe2e2ca2700 25 osd.0 27 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254273+0000 last_rx_front 2019-12-07T19:34:02.254232+0000 2019-12-07T19:34:04.632+0000 7fe2e2ca2700 25 osd.0 27 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254209+0000 last_rx_front 2019-12-07T19:34:02.254194+0000 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_op obc obc(1:41fe8a11:::smithi13140796-45:head rwstate(read n=2 w=0)) 2019-12-07T19:34:04.632+0000 7fe2e2ca2700 25 osd.0 27 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254409+0000 last_rx_front 2019-12-07T19:34:02.254290+0000 2019-12-07T19:34:04.632+0000 7fe2e2ca2700 25 osd.0 27 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254354+0000 last_rx_front 2019-12-07T19:34:02.254301+0000 2019-12-07T19:34:04.632+0000 7fe2e2ca2700 25 osd.0 27 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254368+0000 last_rx_front 2019-12-07T19:34:02.254333+0000 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx 0x558e83c9e300 2019-12-07T19:34:04.632+0000 7fe2e2ca2700 20 osd.0 27 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:04.632+0000 7fe2e2ca2700 20 osd.0 27 sched_scrub not scheduling scrubs due to active recovery 2019-12-07T19:34:04.632+0000 7fe2e2ca2700 10 osd.0 27 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx 1:41fe8a11:::smithi13140796-45:head [read 0~753664,checksum in=4b] ov 21'6 2019-12-07T19:34:04.632+0000 7fe2e2ca2700 20 osd.0 27 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:04.632+0000 7fe2e2ca2700 10 osd.0 27 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op 1:41fe8a11:::smithi13140796-45:head [read 0~753664,checksum in=4b] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op read 0~753664 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_read 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] async_read noted for 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op checksum in=4b 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_checksum 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_checksum: async_read noted for 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 0(0) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 4(1) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] start_read_op: starting ReadOp(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_read_op: starting read ReadOp(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8402f180 con 0x558e802a5400 2019-12-07T19:34:04.632+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.2s2 27/26 ECSubRead(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e81a03680 con 0x558e818d7c00 2019-12-07T19:34:04.632+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpRead(1.2s5 27/26 ECSubRead(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e83c72c80 con 0x558e818d7000 2019-12-07T19:34:04.632+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.2s1 27/26 ECSubRead(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e819b6280 con 0x558e818fe400 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_read_op: started ReadOp(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, []))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=0(0),1(2),3(5),4(1)) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81943080 finish 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8) prio 63 cost 4 e27) pg 0x558e81075000 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e819438c0 prio 63 cost 4 latency 2.978697 osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: osd_op(client.4540.0:197 1.2s0 1.88517f82 (undecoded) ondisk+read+known_if_redirected e24) v8 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_op: op osd_op(client.4540.0:197 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~753664,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e24) v8 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_op osd_op(client.4540.0:197 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~753664,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e24) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] readable_with_acting: locs:0(0),1(2),3(5),4(1),5(4) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_object_context: found obc in cache: 0x558e8115b440 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_object_context: 0x558e8115b440 1:41fe8a11:::smithi13140796-45:head rwstate(read n=2 w=0) oi: 1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e83dcc900 snapset: 0=[]:{} 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] find_object_context 1:41fe8a11:::smithi13140796-45:head @head oi=1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] readable_with_acting: locs:0(0),1(2),3(5),4(1),5(4) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 25 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_op oi 1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_op obc obc(1:41fe8a11:::smithi13140796-45:head rwstate(read n=3 w=0)) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx 0x558e83c49800 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx 1:41fe8a11:::smithi13140796-45:head [read 0~753664,checksum in=4b] ov 21'6 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op 1:41fe8a11:::smithi13140796-45:head [read 0~753664,checksum in=4b] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op read 0~753664 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_read 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] async_read noted for 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op checksum in=4b 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_checksum 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_checksum: async_read noted for 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_ops error: (115) Operation now in progress 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 0(0) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 4(1) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] start_read_op: starting ReadOp(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_read_op: starting read ReadOp(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, []))}, priority=127, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e819b6780 con 0x558e802a5400 2019-12-07T19:34:04.632+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.2s2 27/26 ECSubRead(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8191b900 con 0x558e818d7c00 2019-12-07T19:34:04.632+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558e8402f180 con 0x558e802a5400 2019-12-07T19:34:04.632+0000 7fe2d427f700 15 osd.0 27 enqueue_op 0x558e83edd4a0 prio 127 cost 0 latency 0.000507 epoch 27 MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.632+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpRead(1.2s5 27/26 ECSubRead(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e83b08500 con 0x558e818d7000 2019-12-07T19:34:04.632+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.2s1 27/26 ECSubRead(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e83b08280 con 0x558e818fe400 2019-12-07T19:34:04.632+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558e819b6780 con 0x558e802a5400 2019-12-07T19:34:04.632+0000 7fe2d427f700 15 osd.0 27 enqueue_op 0x558e83edd600 prio 127 cost 0 latency 0.000029 epoch 27 MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.632+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_read_op: started ReadOp(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, []))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=0(0),1(2),3(5),4(1)) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e819438c0 finish 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edd340 prio 127 cost 0 latency 0.001152 MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.632+0000 7fe2caa6c700 25 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.632+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 135 ==== MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=103, reqid=client.4540.0:191, at_version=27'18, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2 ==== 1766+0+0 (unknown 0 0 0) 0x558e83cbe800 con 0x558e818d7c00 2019-12-07T19:34:04.632+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 2019-12-07T19:34:04.632+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84176840 prio 127 cost 0 latency 0.000029 epoch 27 MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=103, reqid=client.4540.0:191, at_version=27'18, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~2e000 size 0x2e000 (188416) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.632+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=103, reqid=client.4540.0:191, at_version=27'18, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840308c0 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84030930 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84030850 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840307e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031260 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031110 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840312d0 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031180 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031340 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840311f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840313b0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031420 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031490 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031500 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031570 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840315e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031650 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840316c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031730 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840317a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031810 blob([0x22a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff29a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff2c40 blob([0x22c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff25b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031880 blob([0x22e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840318f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031960 blob([0x230000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840319d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031a40 blob([0x232000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031ab0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031b20 blob([0x234000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031b90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031c00 blob([0x236000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031c70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031ce0 blob([0x238000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031d50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.632+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031dc0 blob([0x23a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031e30 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031ea0 blob([0x23c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031f10 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031f80 blob([0x23e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede000 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ede070 blob([0x240000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede0e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ede150 blob([0x242000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede1c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ede070 blob([0x240000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede0e0 sbid 0x0)) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ede150 blob([0x242000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede1c0 sbid 0x0)) need {<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2c000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff2c40 blob([0x22c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff25b0 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84030850 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840307e0 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e840308c0 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84030930 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031260 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031110 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e840312d0 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031180 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031340 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840311f0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e840313b0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031420 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031490 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031500 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031570 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840315e0 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031650 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840316c0 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031730 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840317a0 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031810 blob([0x22a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff29a0 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031880 blob([0x22e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840318f0 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031960 blob([0x230000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840319d0 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031a40 blob([0x232000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031ab0 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031b20 blob([0x234000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031b90 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031c00 blob([0x236000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031c70 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031ce0 blob([0x238000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031d50 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031dc0 blob([0x23a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031e30 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031ea0 blob([0x23c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031f10 sbid 0x0)) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84031f80 blob([0x23e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede000 sbid 0x0)) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=103, reqid=client.4540.0:191, at_version=27'18, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=103, reqid=client.4540.0:191, at_version=27'18, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) pg 0x558e841a4000 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e84176840 prio 127 cost 0 latency 0.000315 MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=103, reqid=client.4540.0:191, at_version=27'18, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2 pg pg[1.8s5( v 27'17 lc 21'3 (0'0,27'17] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active m=7 mbc={} ps=[2~1]] 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'17 lc 21'3 (0'0,27'17] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active m=7 mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=103, reqid=client.4540.0:191, at_version=27'18, trim_to=0'0, roll_forward_to=0'0, backfill_or_async_recovery)) v2 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'17 lc 21'3 (0'0,27'17] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'17 active m=7 mbc={} ps=[2~1]] append_log log((0'0,27'17], crt=27'17) [27'18 (27'17) modify 1:10779340:::smithi13140796-32:head by client.4540.0:191 2019-12-07T19:34:01.653811+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'17 crt=27'18 active m=7 mbc={} ps=[2~1]] add_log_entry 27'18 (27'17) modify 1:10779340:::smithi13140796-32:head by client.4540.0:191 2019-12-07T19:34:01.653811+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'17 crt=27'18 active m=7 mbc={} ps=[2~1]] append_log approx pg log length = 18 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'17 crt=27'18 active m=7 mbc={} ps=[2~1]] append_log transaction_applied = 0 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'17 crt=27'18 active m=7 mbc={} ps=[2~1]] append_log 0(5) is async_recovery or backfill target 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.633+0000 7fe2cba6e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'18, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e8417b200 seq 6 2019-12-07T19:34:04.633+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000027.00000000000000000018' <- 0000000027.00000000000000000018 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._fastinfo' <- _fastinfo 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.missing/0000000000000001.80EE9C20.head.smithi13140796-32..' <- missing/0000000000000001.80EE9C20.head.smithi13140796-32.. 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417b200 cost 5457 (1 ios * 4000 + 1457 bytes) 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417b200 onodes shared_blobs 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417b200 allocated 0x[] released 0x[] 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 prepare 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417b200 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 io_done 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'17 crt=27'18 active m=7 mbc={} ps=[2~1]] op_applied version 27'18 2019-12-07T19:34:04.633+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e84176840 finish 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.633+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.633+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.633+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1281 < min_flush_size 524288 2019-12-07T19:34:04.633+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.633+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1304 < min_flush_size 524288 2019-12-07T19:34:04.633+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1304 < min_flush_size 524288 2019-12-07T19:34:04.633+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.633+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xa6bea~518 to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.633+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xa6bea 0xa6bea~518 to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.633+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.633+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~80000 x_off 0x76bea 2019-12-07T19:34:04.633+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xbea 2019-12-07T19:34:04.633+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.633+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x102 and padding block with 0xefe 2019-12-07T19:34:04.633+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xa7102 2019-12-07T19:34:04.633+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.633+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.633+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.633+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.633+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000169118s (0.000000077s flush + 0.000169041s kv commit) 2019-12-07T19:34:04.633+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.633+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.633+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417b200> 2019-12-07T19:34:04.633+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.633+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 kv_submitted 2019-12-07T19:34:04.633+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417b200 2019-12-07T19:34:04.633+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 finishing 2019-12-07T19:34:04.633+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417b200 onodes 2019-12-07T19:34:04.633+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b200 done 2019-12-07T19:34:04.633+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.633+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417b200 [] 2019-12-07T19:34:04.633+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.633+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.633+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ede070 blob([0x240000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede0e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ede0f8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e84032120 space 0x558e83ede0f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ede150 blob([0x242000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede1c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ede1d8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83d118c0 space 0x558e83ede1d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff2c40 blob([0x22c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff25b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff25c8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e840325a0 space 0x558e83ff25c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84030850 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840307e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e840307f8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83d10f30 space 0x558e840307f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e840308c0 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84030930 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84030948 in 0x558e802195e0) _discard 0x1000~1000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83d10870 space 0x558e84030948 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031260 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031110 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031128 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83d10bd0 space 0x558e84031128 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e840312d0 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031180 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031198 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e84033c20 space 0x558e84031198 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031340 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840311f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031208 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e81171ef0 space 0x558e84031208 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e840313b0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031420 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031438 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e81170630 space 0x558e84031438 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031490 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031500 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031518 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e811706c0 space 0x558e84031518 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031570 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840315e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e840315f8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e81170fc0 space 0x558e840315f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031650 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840316c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e840316d8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e81171cb0 space 0x558e840316d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031730 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840317a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e840317b8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e811703f0 space 0x558e840317b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031810 blob([0x22a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff29a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff29b8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e81170ab0 space 0x558e83ff29b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031880 blob([0x22e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840318f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031908 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e811718c0 space 0x558e84031908 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031960 blob([0x230000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840319d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e840319e8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e81170000 space 0x558e840319e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031a40 blob([0x232000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031ab0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031ac8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e81171e60 space 0x558e84031ac8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031b20 blob([0x234000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031b90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031ba8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e81170900 space 0x558e84031ba8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031c00 blob([0x236000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031c70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031c88 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e81170cf0 space 0x558e84031c88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031ce0 blob([0x238000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031d50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031d68 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e81171b00 space 0x558e84031d68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031dc0 blob([0x23a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031e30 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031e48 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 27/26 ECSubWriteReply(tid=103, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e8118ea00 con 0x558e818d7c00 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e81170c60 space 0x558e84031e48 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031ea0 blob([0x23c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031f10 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e84031f28 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e81170360 space 0x558e84031f28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84031f80 blob([0x23e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede000 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ede018 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83c8e870 space 0x558e83ede018 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.633+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.633+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 = 188416 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read read request=188416 r=188416 len=188416 2019-12-07T19:34:04.633+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_hash_info: Getting attr on 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.633+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_hash_info: not in cache 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.633+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:34:04.633+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_hash_info: found on disk, size 188416 2019-12-07T19:34:04.633+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key 2019-12-07T19:34:04.633+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key = 0 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read: Checking hash of 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.633+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 -- 0x558e841da000 con 0x558e802a5400 2019-12-07T19:34:04.633+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edd340 finish 2019-12-07T19:34:04.633+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558e841da000 con 0x558e802a5400 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.633+0000 7fe2d427f700 15 osd.0 27 enqueue_op 0x558e83edd760 prio 127 cost 0 latency 0.000014 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:04.633+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.633+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.633+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.633+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.633+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edd4a0 prio 127 cost 0 latency 0.001594 MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.633+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.633+0000 7fe2c6a64700 25 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.633+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 2019-12-07T19:34:04.633+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~2e000 size 0x2e000 (188416) 2019-12-07T19:34:04.633+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.633+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840308c0 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84030930 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.633+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84030850 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840307e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.633+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031260 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031110 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.633+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840312d0 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.633+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031340 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840311f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.633+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840313b0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.633+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031490 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031500 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.633+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031570 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840315e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.633+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031650 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840316c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.633+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031730 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840317a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031810 blob([0x22a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff29a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff2c40 blob([0x22c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff25b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031880 blob([0x22e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840318f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031960 blob([0x230000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840319d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031a40 blob([0x232000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031ab0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031b20 blob([0x234000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031b90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031c00 blob([0x236000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031c70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031ce0 blob([0x238000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031d50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031dc0 blob([0x23a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031e30 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031ea0 blob([0x23c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031f10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031f80 blob([0x23e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede000 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ede070 blob([0x240000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede0e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ede150 blob([0x242000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede1c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 = 188416 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read read request=188416 r=188416 len=188416 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_hash_info: Getting attr on 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_hash_info: not in cache 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_hash_info: found on disk, size 188416 2019-12-07T19:34:04.634+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key = 0 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read: Checking hash of 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.634+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 -- 0x558e84242780 con 0x558e802a5400 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edd4a0 finish 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.634+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558e84242780 con 0x558e802a5400 2019-12-07T19:34:04.634+0000 7fe2d427f700 15 osd.0 27 enqueue_op 0x558e83edd080 prio 127 cost 0 latency 0.000020 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 2019-12-07T19:34:04.634+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edd600 prio 127 cost 0 latency 0.001371 MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=0,188416,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.634+0000 7fe2c6a64700 25 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.634+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~2e000 size 0x2e000 (188416) 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840308c0 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84030930 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84030850 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840307e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031260 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031110 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840312d0 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031340 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840311f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840313b0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031490 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031500 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031570 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840315e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031650 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840316c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031730 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840317a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031810 blob([0x22a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff29a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff2c40 blob([0x22c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff25b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031880 blob([0x22e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840318f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031960 blob([0x230000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840319d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031a40 blob([0x232000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031ab0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031b20 blob([0x234000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031b90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031c00 blob([0x236000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031c70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031ce0 blob([0x238000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031d50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031dc0 blob([0x23a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031e30 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031ea0 blob([0x23c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031f10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031f80 blob([0x23e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede000 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ede070 blob([0x240000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede0e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ede150 blob([0x242000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede1c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 0x0~2e000 = 188416 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read read request=188416 r=188416 len=188416 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_hash_info: Getting attr on 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_hash_info: not in cache 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_hash_info: found on disk, size 188416 2019-12-07T19:34:04.634+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key = 0 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read: Checking hash of 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.634+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 -- 0x558e8402f180 con 0x558e802a5400 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edd600 finish 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.634+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558e8402f180 con 0x558e802a5400 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.634+0000 7fe2d427f700 15 osd.0 27 enqueue_op 0x558e83edcc60 prio 127 cost 0 latency 0.000015 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:34:04.634+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.634+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.634+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.634+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edd760 prio 127 cost 0 latency 0.000536 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.634+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:04.634+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=17, attrs_read=0) 2019-12-07T19:34:04.634+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, [0(0),188416]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=1(2),3(5),4(1)) 2019-12-07T19:34:04.634+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edd760 finish 2019-12-07T19:34:04.634+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.634+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.634+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.634+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.634+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edd080 prio 127 cost 0 latency 0.000373 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.634+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 2019-12-07T19:34:04.634+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=18, attrs_read=0) 2019-12-07T19:34:04.634+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, [0(0),188416]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=1(2),3(5),4(1)) 2019-12-07T19:34:04.634+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edd080 finish 2019-12-07T19:34:04.634+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edcc60 prio 127 cost 0 latency 0.000192 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=19, attrs_read=0) 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, [0(0),188416]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=1(2),3(5),4(1)) 2019-12-07T19:34:04.634+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edcc60 finish 2019-12-07T19:34:04.634+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.635+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 78 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 ==== 188581+0+0 (unknown 0 0 0) 0x558e84242500 con 0x558e818fe400 2019-12-07T19:34:04.635+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e81942b00 prio 127 cost 0 latency 0.000021 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:04.635+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.635+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.635+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.635+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.635+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.635+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81942b00 prio 127 cost 0 latency 0.000042 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.635+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:04.635+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=17, attrs_read=0) 2019-12-07T19:34:04.635+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, [0(0),188416, 4(1),188416]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=1(2),3(5)) 2019-12-07T19:34:04.635+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81942b00 finish 2019-12-07T19:34:04.635+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.635+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 79 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 ==== 188581+0+0 (unknown 0 0 0) 0x558e83b08280 con 0x558e818fe400 2019-12-07T19:34:04.635+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e81943600 prio 127 cost 0 latency 0.000014 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 2019-12-07T19:34:04.635+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.635+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.635+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.635+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.635+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.635+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e81943600 prio 127 cost 0 latency 0.000037 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.635+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 2019-12-07T19:34:04.635+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=18, attrs_read=0) 2019-12-07T19:34:04.635+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, [0(0),188416, 4(1),188416]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=1(2),3(5)) 2019-12-07T19:34:04.635+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e81943600 finish 2019-12-07T19:34:04.635+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.636+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 80 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 ==== 188581+0+0 (unknown 0 0 0) 0x558e819b6280 con 0x558e818fe400 2019-12-07T19:34:04.636+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e81942420 prio 127 cost 0 latency 0.000018 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:34:04.636+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.636+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.636+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.636+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.636+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.636+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81942420 prio 127 cost 0 latency 0.000527 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.636+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:34:04.636+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=19, attrs_read=0) 2019-12-07T19:34:04.636+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, [0(0),188416, 4(1),188416]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=1(2),3(5)) 2019-12-07T19:34:04.637+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81942420 finish 2019-12-07T19:34:04.637+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.638+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 75 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 ==== 188581+0+0 (unknown 0 0 0) 0x558e83b08500 con 0x558e818d7000 2019-12-07T19:34:04.638+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e841769a0 prio 127 cost 0 latency 0.000025 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:04.638+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.638+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 76 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 ==== 188581+0+0 (unknown 0 0 0) 0x558e8402f400 con 0x558e818d7000 2019-12-07T19:34:04.638+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84176b00 prio 127 cost 0 latency 0.000015 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 2019-12-07T19:34:04.638+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.638+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.638+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.638+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.638+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e841769a0 prio 127 cost 0 latency 0.000179 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=17, attrs_read=0) 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, [0(0),188416, 3(5),188416, 4(1),188416]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=1(2)) 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e841769a0 finish 2019-12-07T19:34:04.638+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.638+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.638+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.638+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e84176b00 prio 127 cost 0 latency 0.000140 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=18, attrs_read=0) 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, [0(0),188416, 3(5),188416, 4(1),188416]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=1(2)) 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e84176b00 finish 2019-12-07T19:34:04.638+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.638+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 77 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 ==== 188581+0+0 (unknown 0 0 0) 0x558e8402fb80 con 0x558e818d7000 2019-12-07T19:34:04.638+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84176c60 prio 127 cost 0 latency 0.000014 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:34:04.638+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.638+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.638+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.638+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.638+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e84176c60 prio 127 cost 0 latency 0.000067 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=19, attrs_read=0) 2019-12-07T19:34:04.638+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, [0(0),188416, 3(5),188416, 4(1),188416]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=1(2)) 2019-12-07T19:34:04.638+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e84176c60 finish 2019-12-07T19:34:04.638+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.638+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 78 ==== MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=53, reqid=client.4540.0:215, at_version=27'6, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 96798+0+0 (unknown 0 0 0) 0x558e83acb800 con 0x558e818d7000 2019-12-07T19:34:04.638+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84176dc0 prio 127 cost 0 latency 0.000037 epoch 27 MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=53, reqid=client.4540.0:215, at_version=27'6, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.638+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=53, reqid=client.4540.0:215, at_version=27'6, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.638+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.638+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=53, reqid=client.4540.0:215, at_version=27'6, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.638+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.638+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=53, reqid=client.4540.0:215, at_version=27'6, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) pg 0x558e819e4000 2019-12-07T19:34:04.638+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e84176dc0 prio 127 cost 0 latency 0.000099 MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=53, reqid=client.4540.0:215, at_version=27'6, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.1s1( v 27'5 (0'0,27'5] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.638+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'5 (0'0,27'5] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=53, reqid=client.4540.0:215, at_version=27'6, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.638+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'5 (0'0,27'5] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'5], crt=21'3) [27'6 (27'5) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:215 2019-12-07T19:34:01.670014+0000 0 ObjectCleanRegions clean_offsets: [0~884736,1261568~18446744073708290047], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.638+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'6 (0'0,27'6] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'5 crt=21'3 lcod 27'5 active+remapped mbc={} ps=[2~1]] add_log_entry 27'6 (27'5) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:215 2019-12-07T19:34:01.670014+0000 0 ObjectCleanRegions clean_offsets: [0~884736,1261568~18446744073708290047], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.638+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'6 (0'0,27'6] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'5 crt=21'3 lcod 27'5 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 6 2019-12-07T19:34:04.638+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'6 (0'0,27'6] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'5 crt=21'3 lcod 27'5 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.638+0000 7fe2cb26d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.638+0000 7fe2cb26d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'6, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e819a0000 seq 19 2019-12-07T19:34:04.639+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2 keys 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2 keys = 0 2019-12-07T19:34:04.639+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:04.639+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x36000~17000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 1#1:891f0d13:::smithi13140796-14:head# 0x36000~17000 - have 0x36000 (221184) bytes fadvise_flags 0x0 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x36000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0000 12 blobs 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 fbmap_alloc 0x558e80233300 allocate 0x17000/1000,17000,0 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x296000~e000/1000,17000,0 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x309000~9000/1000,17000,0 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x296000~e000,0x309000~9000] 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423d5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03f80 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d5e0 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 27 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 23'7 (0'0,23'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) e27/27) v1 ==== 1178+0+0 (unknown 0 0 0) 0x558e83c9f100 con 0x558e818d6000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e83d03f80) get_ref 0x0~2000 Blob(0x558e83d03f80 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d5e0 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e83d03f80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558e83d03f80 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d5e0 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423d5f8 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423d570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423d500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.639+0000 7fe2e753b700 15 osd.0 27 enqueue_peering_evt 1.5s4 epoch_sent: 27 epoch_requested: 27 MInfoRec from 2(0) info: 1.5s0( v 23'7 (0'0,23'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.078908034s ub 44.078908034s int 16s) 2019-12-07T19:34:04.639+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 2(0) info: 1.5s0( v 23'7 (0'0,23'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.078908034s ub 44.078908034s int 16s)) prio 255 cost 10 e27) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423d570 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d500 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423d570) get_ref 0x0~2000 Blob(0x558e8423d570 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d500 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423d570) get_ref init 0x2000, 1000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558e8423d570 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d500 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423d518 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423d490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423d420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423d490 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d420 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423d490) get_ref 0x0~2000 Blob(0x558e8423d490 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d420 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423d490) get_ref init 0x2000, 1000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558e8423d490 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d420 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423d438 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423d3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423d340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423d3b0 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d340 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423d3b0) get_ref 0x0~2000 Blob(0x558e8423d3b0 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d340 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423d3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558e8423d3b0 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d340 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423d358 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423d2d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423d260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423d2d0 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d260 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423d2d0) get_ref 0x0~2000 Blob(0x558e8423d2d0 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d260 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423d2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558e8423d2d0 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d260 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423d278 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423d1f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423d180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423d1f0 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d180 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423d1f0) get_ref 0x0~2000 Blob(0x558e8423d1f0 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d180 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423d1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558e8423d1f0 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d180 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423d198 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423d110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423d0a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423d110 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d0a0 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423d110) get_ref 0x0~2000 Blob(0x558e8423d110 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d0a0 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423d110) get_ref init 0x2000, 1000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558e8423d110 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d0a0 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423d0b8 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423d030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423cfc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423d030 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423cfc0 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423d030) get_ref 0x0~2000 Blob(0x558e8423d030 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423cfc0 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423d030) get_ref init 0x2000, 1000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558e8423d030 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cfc0 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423cfd8 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423cf50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423cee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423cf50 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423cee0 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423cf50) get_ref 0x0~2000 Blob(0x558e8423cf50 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423cee0 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423cf50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558e8423cf50 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cee0 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423cef8 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423ce70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423ce00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423ce70 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423ce00 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423ce70) get_ref 0x0~2000 Blob(0x558e8423ce70 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423ce00 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e8423ce70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558e8423ce70 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ce00 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423ce18 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f95c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a91570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f95c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a91570 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e83f95c70) get_ref 0x0~2000 Blob(0x558e83f95c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a91570 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e83f95c70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558e83f95c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a91570 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e81a91588 in 0x558e80219500) _discard 0x0~2000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a91500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423d650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a91500 blob([0x311000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d650 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e81a91500) get_ref 0x0~1000 Blob(0x558e81a91500 blob([0x311000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423d650 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.blob(0x558e81a91500) get_ref init 0x1000, 1000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4c000~1000: 0x0~1000 Blob(0x558e81a91500 blob([0x311000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8423d650 sbid 0x0)) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423d668 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x4d000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) dirty_range mark inline shard dirty 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x36000~17000 = 0 2019-12-07T19:34:04.639+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.639+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.0000000027.00000000000000000006' <- 0000000027.00000000000000000006 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0000 cost 148330 (13 ios * 4000 + 96330 bytes) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0000 onodes 0x558e811dd680 shared_blobs 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) update 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) update inline shard 797 bytes from 39 extents 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:891f0d13:::smithi13140796-14:head# is 55265 (54462 bytes onode + 2 bytes spanning blobs + 801 bytes inline extents) 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0000 allocated 0x[296000~e000,309000~9000] released 0x[] 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 freelist allocate 0x296000~e000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 freelist allocate 0x309000~9000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 prepare 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e819a0000 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'6 (0'0,27'6] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'5 crt=21'3 lcod 27'5 active+remapped mbc={} ps=[2~1]] op_applied version 27'6 2019-12-07T19:34:04.639+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e84176dc0 finish 2019-12-07T19:34:04.639+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.639+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.639+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 2(0) info: 1.5s0( v 23'7 (0'0,23'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.078908034s ub 44.078908034s int 16s)) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.639+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.639+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MInfoRec from 2(0) info: 1.5s0( v 23'7 (0'0,23'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.078908034s ub 44.078908034s int 16s)) prio 255 cost 10 e27) pg 0x558e83d07000 2019-12-07T19:34:04.639+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 lcod 0'0 active m=3 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MInfoRec from 2(0) info: 1.5s0( v 23'7 (0'0,23'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) pg_lease(ru 44.078908034s ub 44.078908034s int 16s) 2019-12-07T19:34:04.639+0000 7fe2c9269700 20 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 lcod 0'0 active m=3 mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26 2019-12-07T19:34:04.639+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:04.639+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:04.639+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e83d4cf00 seq 3 2019-12-07T19:34:04.639+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.639+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:04.639+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.639+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4cf00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:04.639+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4cf00 onodes shared_blobs 2019-12-07T19:34:04.639+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4cf00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.639+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 prepare 2019-12-07T19:34:04.639+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4cf00 2019-12-07T19:34:04.639+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 io_done 2019-12-07T19:34:04.639+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.639+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.639+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.639+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:04.639+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:04.639+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.639+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1046 < min_flush_size 524288 2019-12-07T19:34:04.639+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.639+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xa7102~416 to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.639+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xa7102 0xa7102~416 to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.639+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.639+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~80000 x_off 0x77102 2019-12-07T19:34:04.639+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x102 2019-12-07T19:34:04.639+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.639+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x518 and padding block with 0xae8 2019-12-07T19:34:04.640+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xa7518 2019-12-07T19:34:04.640+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.640+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.640+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.640+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 136 ==== MOSDECSubOpWrite(1.9s3 27/26 ECSubWrite(tid=104, reqid=client.4540.0:209, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 150788+0+0 (unknown 0 0 0) 0x558e83ac8000 con 0x558e818d7c00 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84176f20 prio 127 cost 0 latency 0.000042 epoch 27 MOSDECSubOpWrite(1.9s3 27/26 ECSubWrite(tid=104, reqid=client.4540.0:209, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 27/26 ECSubWrite(tid=104, reqid=client.4540.0:209, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.641+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.001485461s (0.000000141s flush + 0.001485320s kv commit) 2019-12-07T19:34:04.641+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4cf00> 2019-12-07T19:34:04.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.641+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 kv_submitted 2019-12-07T19:34:04.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4cf00 2019-12-07T19:34:04.641+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 finishing 2019-12-07T19:34:04.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4cf00 onodes 2019-12-07T19:34:04.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4cf00 done 2019-12-07T19:34:04.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:04.641+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4cf00 [] 2019-12-07T19:34:04.641+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.641+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 137 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 ==== 188581+0+0 (unknown 0 0 0) 0x558e8191b900 con 0x558e818d7c00 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84177080 prio 127 cost 0 latency 0.000014 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 27/26 ECSubWrite(tid=104, reqid=client.4540.0:209, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 27/26 ECSubWrite(tid=104, reqid=client.4540.0:209, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) pg 0x558e8192b000 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 osd.0 27 dequeue_op 0x558e84176f20 prio 127 cost 0 latency 0.000297 MOSDECSubOpWrite(1.9s3 27/26 ECSubWrite(tid=104, reqid=client.4540.0:209, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.9s3 27/26 ECSubWrite(tid=104, reqid=client.4540.0:209, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 138 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 ==== 188581+0+0 (unknown 0 0 0) 0x558e81a03680 con 0x558e818d7c00 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 snap_mapper.add_oid 1:9694a0e5:::smithi13140796-3:4 4 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e841771e0 prio 127 cost 0 latency 0.000011 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 snap_mapper.get_snaps 1:9694a0e5:::smithi13140796-3:4 got.empty() 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 snap_mapper.set_snaps 1:9694a0e5:::smithi13140796-3:4 4 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 snap_mapper.set_snaps set OBJ_.3_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 snap_mapper.add_oid set SNA_1_0000000000000004_.3_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 21'6 (0'0,21'6] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] append_log log((0'0,21'6], crt=21'6) [27'7 (21'3) clone 1:9694a0e5:::smithi13140796-3:4 by unknown.0.0:0 2019-12-07T19:33:58.718796+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,27'8 (21'3) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:209 0.000000 0 ObjectCleanRegions clean_offsets: [589824~18446744073708961791], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.641+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 28 ==== MOSDPGRecoveryDelete(1.5s4 e27,26 1:a7686634:::smithi13140796-35:head,23'7) v2 ==== 123+0+0 (unknown 0 0 0) 0x558e84053600 con 0x558e818d6000 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'7 (0'0,27'7] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] add_log_entry 27'7 (21'3) clone 1:9694a0e5:::smithi13140796-3:4 by unknown.0.0:0 2019-12-07T19:33:58.718796+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.641+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e83b0ab00 prio 196 cost 1000 latency 0.000021 epoch 27 MOSDPGRecoveryDelete(1.5s4 e27,26 1:a7686634:::smithi13140796-35:head,23'7) v2 2019-12-07T19:34:04.641+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGRecoveryDelete(1.5s4 e27,26 1:a7686634:::smithi13140796-35:head,23'7) v2) prio 196 cost 1000 e27) 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] add_log_entry 27'8 (21'3) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:209 0.000000 0 ObjectCleanRegions clean_offsets: [589824~18446744073708961791], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.641+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 29 ==== MRecoveryReserve(1.5s4 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e8403e780 con 0x558e818d6000 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 8 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 139 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 ==== 188581+0+0 (unknown 0 0 0) 0x558e8417c000 con 0x558e818d7c00 2019-12-07T19:34:04.641+0000 7fe2e753b700 15 osd.0 27 enqueue_peering_evt 1.5s4 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.641+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84177340 prio 127 cost 0 latency 0.000014 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.641+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 140 ==== MRecoveryReserve(1.2s0 GRANT e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e819b0780 con 0x558e818d7c00 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e8417af00 seq 20 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) get_onode oid 3#1:9694a0e5:::smithi13140796-3:4# key 0x8380000000000000019694a0e521736d'ithi13140796-3!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558e83d07000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) r -2 v.len 0 2019-12-07T19:34:04.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _clone 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# -> 3#1:9694a0e5:::smithi13140796-3:4# 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1125 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.onode(0x558e81a03b80).flush flush done 2019-12-07T19:34:04.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x0 2019-12-07T19:34:04.641+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 lcod 0'0 active m=3 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_clone_range 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# -> 3#1:9694a0e5:::smithi13140796-3:4# 0x0~3000 -> 0x0~3000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.extentmap(0x558e81a03c88) dup src 0x0~2000: 0x0~2000 Blob(0x558e8194e690 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10258 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) make_blob_shared Blob(0x558e8194e690 blob([0x37000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) make_blob_shared now Blob(0x558e8194e690 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1)))) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.extentmap(0x558e81a03c88) dup new Blob(0x558e83d03420 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=2)))) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83d03420) get_ref 0x0~2000 Blob(0x558e83d03420 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=2)))) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83d03420) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7fe2c5261700 5 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 lcod 0'0 active m=3 mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.642336+0000 3 0.000172 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.extentmap(0x558e81a03c88) dup dst 0x0~2000: 0x0~2000 Blob(0x558e83d03420 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=2)))) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.extentmap(0x558e81a03c88) dup src 0x2000~1000: 0x0~1000 Blob(0x558e8194f340 blob([0x39000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194f3b0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10259 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) make_blob_shared Blob(0x558e8194f340 blob([0x39000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194f3b0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) make_blob_shared now Blob(0x558e8194f340 blob([0x39000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=1)))) 2019-12-07T19:34:04.641+0000 7fe2c5261700 5 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 lcod 0'0 active m=3 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.extentmap(0x558e81a03c88) dup new Blob(0x558e83d03260 blob([0x39000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=2)))) 2019-12-07T19:34:04.641+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83d03260) get_ref 0x0~1000 Blob(0x558e83d03260 blob([0x39000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=2)))) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83d03260) get_ref init 0x1000, 1000 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.extentmap(0x558e81a03c88) dup dst 0x2000~1000: 0x0~1000 Blob(0x558e83d03260 blob([0x39000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=2)))) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.extentmap(0x558e81a03c88) dirty_range mark inline shard dirty 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.extentmap(0x558e8402f288) dirty_range mark inline shard dirty 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGRecoveryDelete(1.5s4 e27,26 1:a7686634:::smithi13140796-35:head,23'7) v2) prio 196 cost 1000 e27) queued 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _clone 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# -> 3#1:9694a0e5:::smithi13140796-3:4# = 0 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGRecoveryDelete(1.5s4 e27,26 1:a7686634:::smithi13140796-35:head,23'7) v2) prio 196 cost 1000 e27) pg 0x558e83d07000 2019-12-07T19:34:04.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rmattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# snapset 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rmattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# snapset = 0 2019-12-07T19:34:04.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 1 keys 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 1 keys = 0 2019-12-07T19:34:04.641+0000 7fe2c5261700 10 osd.0 27 dequeue_op 0x558e83b0ab00 prio 196 cost 1000 latency 0.000179 MOSDPGRecoveryDelete(1.5s4 e27,26 1:a7686634:::smithi13140796-35:head,23'7) v2 pg pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 lcod 0'0 active m=3 mbc={} ps=[2~1]] 2019-12-07T19:34:04.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# hinfo_key (42 bytes) 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) get_onode oid 3#1:9694a0e5:::smithi13140796-3:head#8 key 0x8380000000000000019694a0e521736d'ithi13140796-3!='0xfffffffffffffffe0000000000000008'o' 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 lcod 0'0 active m=3 mbc={} ps=[2~1]] handle_recovery_delete 0x558e83b0ab00 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) r -2 v.len 0 2019-12-07T19:34:04.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# -> 3#1:9694a0e5:::smithi13140796-3:head#8 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 lcod 0'0 active m=3 mbc={} ps=[2~1]] remove_missing_object 1:a7686634:::smithi13140796-35:head 23'7 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# -> 3#1:9694a0e5:::smithi13140796-3:head#8 = 0 2019-12-07T19:34:04.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1126 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# = 0 2019-12-07T19:34:04.641+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:04.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 3 keys 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e8195f800 seq 4 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 3 keys = 0 2019-12-07T19:34:04.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.5s4_head 0x558e8403fe00) get_onode oid 4#1:a7686634:::smithi13140796-35:head# key 0x848000000000000001a76866'4!smithi13140796-35!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:04.641+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x0~24000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:9694a0e5:::smithi13140796-3:head# 0x0~24000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~24000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.5s4_head 0x558e8403fe00) r -2 v.len 0 2019-12-07T19:34:04.641+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 4#1:a7686634:::smithi13140796-35:head# 2019-12-07T19:34:04.641+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195f800 cost 4212 (1 ios * 4000 + 212 bytes) 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195f800 onodes shared_blobs 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195f800 allocated 0x[] released 0x[] 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8417af00 18 blobs 2019-12-07T19:34:04.641+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 prepare 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195f800 2019-12-07T19:34:04.641+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 io_done 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2c5261700 10 osd.0 27 dequeue_op 0x558e83b0ab00 finish 2019-12-07T19:34:04.641+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.641+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.641+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:04.641+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xa7518~2e to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xa7518 0xa7518~2e to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.641+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~80000 x_off 0x77518 2019-12-07T19:34:04.641+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x518 2019-12-07T19:34:04.641+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.641+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x546 and padding block with 0xaba 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 fbmap_alloc 0x558e80233300 allocate 0x24000/1000,24000,0 2019-12-07T19:34:04.641+0000 7fe2c7265700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x312000~24000/1000,24000,0 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x312000~24000] 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d033b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03500 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83d033b0) get_ref 0x0~2000 Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03500 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83d033b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83d03518 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d035e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03570 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83d035e0) get_ref 0x0~2000 Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03570 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83d035e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03570 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83d03588 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d036c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03650 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83d036c0) get_ref 0x0~2000 Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03650 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83d036c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03650 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83d03668 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d037a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841da000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da000 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83d037a0) get_ref 0x0~2000 Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da000 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e83d037a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da000 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da018 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841da070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841da0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da0e0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e841da070) get_ref 0x0~2000 Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da0e0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e841da070) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da0e0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da0f8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841da150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841da1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da1c0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e841da150) get_ref 0x0~2000 Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da1c0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e841da150) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da1c0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da1d8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841da230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841da2a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da2a0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e841da230) get_ref 0x0~2000 Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da2a0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e841da230) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da2a0 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da2b8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841da310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841da380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da380 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e841da310) get_ref 0x0~2000 Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da380 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e841da310) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da380 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da398 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841da3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841da460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da460 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e841da3f0) get_ref 0x0~2000 Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da460 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e841da3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da460 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 79 ==== MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=54, reqid=client.4540.0:216, at_version=27'7, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2556+0+0 (unknown 0 0 0) 0x558e83acb000 con 0x558e818d7000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da478 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841da4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841da540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e841774a0 prio 127 cost 0 latency 0.000036 epoch 27 MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=54, reqid=client.4540.0:216, at_version=27'7, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da540 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=54, reqid=client.4540.0:216, at_version=27'7, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e841da4d0) get_ref 0x0~2000 Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da540 sbid 0x0)) 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore.blob(0x558e841da4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.641+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da540 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da558 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841da5b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841da620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da620 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841da5b0) get_ref 0x0~2000 Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da620 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 80 ==== MRecoveryReserve(1.1s1 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e819b0780 con 0x558e818d7000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841da5b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da620 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da638 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.642+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.1s1 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841da690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841da700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.642+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da700 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841da690) get_ref 0x0~2000 Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da700 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841da690) get_ref init 0x2000, 1000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da700 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da718 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841da770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841da7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da7e0 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841da770) get_ref 0x0~2000 Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da7e0 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841da770) get_ref init 0x2000, 1000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da7e0 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da7f8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841da850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841da8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da8c0 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841da850) get_ref 0x0~2000 Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da8c0 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841da850) get_ref init 0x2000, 1000 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da8c0 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558e819e4000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da8d8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841da930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841da9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da9a0 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841da930) get_ref 0x0~2000 Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841da9a0 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841da930) get_ref init 0x2000, 1000 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'6 (0'0,27'6] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 lcod 27'5 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da9a0 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da9b8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841daa10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841daa80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841daa80 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841daa10) get_ref 0x0~2000 Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841daa80 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841daa10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.642+0000 7fe2cb26d700 5 osd.0 pg_epoch: 27 pg[1.1s1( v 27'6 (0'0,27'6] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 lcod 27'5 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.642768+0000 4 0.000098 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841daa80 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841daa98 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.642+0000 7fe2cb26d700 5 osd.0 pg_epoch: 27 pg[1.1s1( v 27'6 (0'0,27'6] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 lcod 27'5 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841daaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841dab60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dab60 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=54, reqid=client.4540.0:216, at_version=27'7, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841daaf0) get_ref 0x0~2000 Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dab60 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841daaf0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=54, reqid=client.4540.0:216, at_version=27'7, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) pg 0x558e819e4000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dab60 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841dab78 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841dabd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841dac40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e841774a0 prio 127 cost 0 latency 0.000151 MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=54, reqid=client.4540.0:216, at_version=27'7, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.1s1( v 27'6 (0'0,27'6] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 lcod 27'5 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dac40 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841dabd0) get_ref 0x0~2000 Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dac40 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.blob(0x558e841dabd0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'6 (0'0,27'6] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 lcod 27'5 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=54, reqid=client.4540.0:216, at_version=27'7, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dac40 sbid 0x0)) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841dac58 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'6 (0'0,27'6] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 lcod 27'5 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'6], crt=21'3) [27'7 (27'6) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:216 2019-12-07T19:34:01.670030+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x24000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.extentmap(0x558e83b08388) dirty_range mark inline shard dirty 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'6 crt=21'3 lcod 27'5 active+remapped mbc={} ps=[2~1]] add_log_entry 27'7 (27'6) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:216 2019-12-07T19:34:01.670030+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'6 crt=21'3 lcod 27'5 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 7 2019-12-07T19:34:04.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x0~24000 = 0 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'6 crt=21'3 lcod 27'5 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.642+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.642+0000 7fe2cb26d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'7, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.642+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.3_0000000000000001.9692507A.4.smithi13140796-3..' <- OBJ_.3_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:04.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:04.642+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e819a0900 seq 20 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.3_0000000000000001.9692507A.4.smithi13140796-3..' <- SNA_1_0000000000000004_.3_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:04.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.642+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 3 keys 2019-12-07T19:34:04.642+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 3 keys = 0 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000027.00000000000000000007' <- 0000000027.00000000000000000007 2019-12-07T19:34:04.642+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000027.00000000000000000008' <- 0000000027.00000000000000000008 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:04.642+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.0000000027.00000000000000000007' <- 0000000027.00000000000000000007 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._fastinfo' <- _fastinfo 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417af00 cost 227812 (19 ios * 4000 + 151812 bytes) 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417af00 onodes 0x558e81a03b80,0x558e83b08280,0x558e8402f180 shared_blobs 0x558e817e8d20,0x558e8194f3b0 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0900 cost 6088 (1 ios * 4000 + 2088 bytes) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.extentmap(0x558e81a03c88) update 3#1:9694a0e5:::smithi13140796-3:head#8 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0900 onodes 0x558e811dd680 shared_blobs 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) update 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.extentmap(0x558e81a03c88) update inline shard 54 bytes from 2 extents 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9694a0e5:::smithi13140796-3:head#8 is 542 (482 bytes onode + 2 bytes spanning blobs + 58 bytes inline extents) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.extentmap(0x558e83b08388) update 3#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.extentmap(0x558e83b08388) update inline shard 345 bytes from 18 extents 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9694a0e5:::smithi13140796-3:head# is 891 (540 bytes onode + 2 bytes spanning blobs + 349 bytes inline extents) 2019-12-07T19:34:04.642+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:891f0d13:::smithi13140796-14:head# is 55265 (54462 bytes onode + 2 bytes spanning blobs + 801 bytes inline extents) 2019-12-07T19:34:04.642+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.642+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.642+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.642+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 aio_wait 2019-12-07T19:34:04.642+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0000 2019-12-07T19:34:04.642+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 io_done 2019-12-07T19:34:04.642+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0900 allocated 0x[] released 0x[] 2019-12-07T19:34:04.642+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_recovery_priority recovery priority is 190 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 prepare 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0900 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 io_done 2019-12-07T19:34:04.642+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MRecoveryReserve(1.2s5 REQUEST prio: 190 e27/27) v3 -- 0x558e819b03c0 con 0x558e818d7000 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'6 crt=21'3 lcod 27'5 active+remapped mbc={} ps=[2~1]] op_applied version 27'7 2019-12-07T19:34:04.642+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.642+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e841774a0 finish 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.642+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.extentmap(0x558e8402f288) update 3#1:9694a0e5:::smithi13140796-3:4# 2019-12-07T19:34:04.642+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore.extentmap(0x558e8402f288) update inline shard 54 bytes from 2 extents 2019-12-07T19:34:04.642+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.642+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9694a0e5:::smithi13140796-3:4# is 469 (409 bytes onode + 2 bytes spanning blobs + 58 bytes inline extents) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2812 is 11 SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=2))) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2813 is 11 SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=2))) 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417af00 allocated 0x[312000~24000] released 0x[] 2019-12-07T19:34:04.642+0000 7fe2c7265700 10 freelist allocate 0x312000~24000 2019-12-07T19:34:04.642+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e84177080 prio 127 cost 0 latency 0.001027 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:04.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 prepare 2019-12-07T19:34:04.642+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8417af00 2019-12-07T19:34:04.642+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=17, attrs_read=0)) v2 2019-12-07T19:34:04.642+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=17, attrs_read=0) 2019-12-07T19:34:04.642+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:04.642+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:04.642+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=5 2019-12-07T19:34:04.642+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=1 2019-12-07T19:34:04.642+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply Complete: ReadOp(tid=17, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, [0(0),188416, 1(2),188416, 3(5),188416, 4(1),188416]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=) 2019-12-07T19:34:04.642+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'6 crt=21'6 lcod 21'6 active+remapped mbc={} ps=[2~1]] op_applied version 27'8 2019-12-07T19:34:04.642+0000 7fe2c7265700 10 osd.0 27 dequeue_op 0x558e84176f20 finish 2019-12-07T19:34:04.642+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.642+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.643+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 aio_wait 2019-12-07T19:34:04.643+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417af00 2019-12-07T19:34:04.643+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 io_done 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xa7546 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.002922076s (0.000000088s flush + 0.002921988s kv commit) 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=2 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2746 < min_flush_size 524288 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 35514 < min_flush_size 524288 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 56529 < min_flush_size 524288 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dd680 had 2 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 2 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 68282 < min_flush_size 524288 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 101050 < min_flush_size 524288 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 112989 < min_flush_size 524288 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dd680 had 1 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 117374 < min_flush_size 524288 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03b80 had 1 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b08280 had 1 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402f180 had 1 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 117397 < min_flush_size 524288 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 117397 < min_flush_size 524288 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xa7546~1ca95 to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xa7546 0xa7546~1ca95 to file(ino 16 size 0xb0000 mtime 2019-12-07T19:34:04.628545+0000 allocated b0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~80000]) 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _allocate len 0x13fdb from 0 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _allocate len 0x13fdb from 1 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x20000/10000,20000,acd1e0000 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd1e0000~20000/10000,20000,acd1e0000 2019-12-07T19:34:04.644+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0xd0000 to include allocated 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 32 (was clean) 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x77546 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x546 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xfdb and padding block with 0x25 2019-12-07T19:34:04.644+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0xd0000 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] objects_read_async_cb: got: {1:41fe8a11:::smithi13140796-45:head=0,{0~753664(753664)}} 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:34:04.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195f800> 2019-12-07T19:34:04.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 kv_submitted 2019-12-07T19:34:04.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195f800 2019-12-07T19:34:04.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 finishing 2019-12-07T19:34:04.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195f800 onodes 2019-12-07T19:34:04.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f800 done 2019-12-07T19:34:04.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:04.645+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195f800 [] 2019-12-07T19:34:04.645+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.645+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] finish_checksum 2019-12-07T19:34:04.645+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx 0x558e8404ae00 2019-12-07T19:34:04.645+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx 1:41fe8a11:::smithi13140796-45:head [read 0~753664 out=753664b,checksum in=4b out=8b,getxattrs] ov 21'6 2019-12-07T19:34:04.645+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op 1:41fe8a11:::smithi13140796-45:head [read 0~753664 out=753664b,checksum in=4b out=8b,getxattrs] 2019-12-07T19:34:04.645+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op read 0~753664 out=753664b 2019-12-07T19:34:04.645+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op checksum in=4b out=8b 2019-12-07T19:34:04.645+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op getxattrs 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx alloc reply 0x558e819b6280 result 0 2019-12-07T19:34:04.645+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:34:04.645+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] log_op_stats osd_op(client.4540.0:195 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~753664 out=753664b,checksum in=4b out=8b,getxattrs out=75b] snapc 0=[] ondisk+read+known_if_redirected e24) v8 inb 0 outb 753747 lat 2.992030 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(4)=3} 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.645+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc3fdb 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.645+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.645+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.645+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.645+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 4 2019-12-07T19:34:04.645+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 4 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 8 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.645+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 lcod 0'0 active m=3 mbc={} ps=[2~1]] on_local_recover: 1:a7686634:::smithi13140796-35:head 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.645+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 lcod 0'0 active m=3 mbc={} ps=[2~1]] got missing 1:a7686634:::smithi13140796-35:head v 23'7 2019-12-07T19:34:04.645+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 27:49 2019-12-07T19:34:04.645+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 lcod 0'0 active m=2 mbc={} ps=[2~1]] last_complete now 21'1 log.complete_to 21'2 2019-12-07T19:34:04.645+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(195 smithi13140796-45 [read 0~753664 out=753664b,checksum out=8b,getxattrs out=75b] v0'0 uv6 ondisk = 0) v8 -- 0x558e819b6280 con 0x558e83d68000 2019-12-07T19:34:04.645+0000 7fe2c9269700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.645+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:04.645+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e83c56600 seq 5 2019-12-07T19:34:04.645+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.645+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x0000000000000462'.missing/0000000000000001.5E6166C2.head.smithi13140796-35..' <- missing/0000000000000001.5E6166C2.head.smithi13140796-35.. 2019-12-07T19:34:04.645+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.645+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.645+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e84177080 finish 2019-12-07T19:34:04.645+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:04.645+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.645+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.645+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.645+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56600 cost 5409 (1 ios * 4000 + 1409 bytes) 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.645+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56600 onodes shared_blobs 2019-12-07T19:34:04.645+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.645+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.645+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 prepare 2019-12-07T19:34:04.645+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56600 2019-12-07T19:34:04.645+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 io_done 2019-12-07T19:34:04.645+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e841771e0 prio 127 cost 0 latency 0.004529 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.645+0000 7fe2c9269700 20 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 lcod 0'0 active m=2 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.645+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=18, attrs_read=0)) v2 2019-12-07T19:34:04.645+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.645+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=18, attrs_read=0) 2019-12-07T19:34:04.646+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:04.646+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:04.646+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=5 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (32) on file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]), flushing log 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.646+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=1 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 32 len 0x49 crc 0xdaae51df) 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x1f000~1000 to file(ino 1 size 0x1f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x1f000 0x1f000~1000 to file(ino 1 size 0x1f000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x20000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x1f000 2019-12-07T19:34:04.646+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply Complete: ReadOp(tid=18, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, [0(0),188416, 1(2),188416, 3(5),188416, 4(1),188416]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=) 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x20000 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000062 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 32 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 3 cleaned 0 in 0.001492730s (0.000003868s flush + 0.001488862s kv commit) 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1185 < min_flush_size 524288 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc3fdb~4b8 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc3fdb 0xc3fdb~4b8 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x93fdb 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xfdb 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x493 and padding block with 0xb6d 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a0000,0x558e819a0900,0x558e8417af00> 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 kv_submitted 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0000 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 finishing 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0000 onodes 0x558e811dd680 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a91588 in 0x558e80219500) _finish_write discard buffer(0x558e84033950 space 0x558e81a91588 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423ce18 in 0x558e80219500) _finish_write discard buffer(0x558e84033a70 space 0x558e8423ce18 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423cef8 in 0x558e80219500) _finish_write discard buffer(0x558e84032a20 space 0x558e8423cef8 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423cfd8 in 0x558e80219500) _finish_write discard buffer(0x558e840337a0 space 0x558e8423cfd8 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423d0b8 in 0x558e80219500) _finish_write discard buffer(0x558e840335f0 space 0x558e8423d0b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423d198 in 0x558e80219500) _finish_write discard buffer(0x558e84033710 space 0x558e8423d198 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423d278 in 0x558e80219500) _finish_write discard buffer(0x558e84033680 space 0x558e8423d278 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423d358 in 0x558e80219500) _finish_write discard buffer(0x558e84033560 space 0x558e8423d358 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423d438 in 0x558e80219500) _finish_write discard buffer(0x558e841ec3f0 space 0x558e8423d438 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423d518 in 0x558e80219500) _finish_write discard buffer(0x558e841ec360 space 0x558e8423d518 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423d5f8 in 0x558e80219500) _finish_write discard buffer(0x558e841ec2d0 space 0x558e8423d5f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423d668 in 0x558e80219500) _finish_write discard buffer(0x558e84032240 space 0x558e8423d668 0x0~1000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0000 done 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0900 kv_submitted 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0000 [] 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 kv_submitted 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0900 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 finishing 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0900 onodes 0x558e811dd680 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0900 done 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0900 [] 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 kv_submitted 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417af00 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 finishing 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417af00 onodes 0x558e81a03b80,0x558e83b08280,0x558e8402f180 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03518 in 0x558e80219a40) _finish_write discard buffer(0x558e84033b00 space 0x558e83d03518 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03588 in 0x558e80219a40) _finish_write discard buffer(0x558e80229a70 space 0x558e83d03588 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03668 in 0x558e80219a40) _finish_write discard buffer(0x558e83c8e900 space 0x558e83d03668 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841da018 in 0x558e80219a40) _finish_write discard buffer(0x558e841ecf30 space 0x558e841da018 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841da0f8 in 0x558e80219a40) _finish_write discard buffer(0x558e841ecfc0 space 0x558e841da0f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 27/26 ECSubWriteReply(tid=53, last_complete=27'6, committed=1, applied=1)) v2 -- 0x558e83fef800 con 0x558e818d7000 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841da1d8 in 0x558e80219a40) _finish_write discard buffer(0x558e841ed050 space 0x558e841da1d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841da2b8 in 0x558e80219a40) _finish_write discard buffer(0x558e841ed0e0 space 0x558e841da2b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841da398 in 0x558e80219a40) _finish_write discard buffer(0x558e841ed170 space 0x558e841da398 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841da478 in 0x558e80219a40) _finish_write discard buffer(0x558e841ed200 space 0x558e841da478 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841da558 in 0x558e80219a40) _finish_write discard buffer(0x558e841ed290 space 0x558e841da558 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841da638 in 0x558e80219a40) _finish_write discard buffer(0x558e841ed320 space 0x558e841da638 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841da718 in 0x558e80219a40) _finish_write discard buffer(0x558e841ed3b0 space 0x558e841da718 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841da7f8 in 0x558e80219a40) _finish_write discard buffer(0x558e841ed440 space 0x558e841da7f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841da8d8 in 0x558e80219a40) _finish_write discard buffer(0x558e841ed4d0 space 0x558e841da8d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841da9b8 in 0x558e80219a40) _finish_write discard buffer(0x558e841ed560 space 0x558e841da9b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841daa98 in 0x558e80219a40) _finish_write discard buffer(0x558e841ed5f0 space 0x558e841daa98 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 27/26 ECSubWriteReply(tid=54, last_complete=27'7, committed=1, applied=1)) v2 -- 0x558e84065400 con 0x558e818d7000 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841dab78 in 0x558e80219a40) _finish_write discard buffer(0x558e841ed680 space 0x558e841dab78 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841dac58 in 0x558e80219a40) _finish_write discard buffer(0x558e841ed710 space 0x558e841dac58 0x0~2000 writing nocache) 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417af00 done 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417af00 [] 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.646+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 27/26 ECSubWriteReply(tid=104, last_complete=27'8, committed=1, applied=1)) v2 -- 0x558e80264000 con 0x558e818d7c00 2019-12-07T19:34:04.646+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.646+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.646+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc4493 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.646+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000290131s (0.000000161s flush + 0.000289970s kv commit) 2019-12-07T19:34:04.646+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56600> 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 kv_submitted 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56600 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 finishing 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56600 onodes 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56600 done 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56600 [] 2019-12-07T19:34:04.646+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.646+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.646+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 lcod 0'0 active m=2 mbc={} ps=[2~1]] recovery_committed_to version 21'1 now ondisk 2019-12-07T19:34:04.646+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDPGRecoveryDeleteReply(1.5s0 e27,26 1:a7686634:::smithi13140796-35:head,23'7) v2 -- 0x558e81955600 con 0x558e818d6000 2019-12-07T19:34:04.646+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.646+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] objects_read_async_cb: got: {1:41fe8a11:::smithi13140796-45:head=0,{0~753664(753664)}} 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] finish_checksum 2019-12-07T19:34:04.648+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx 0x558e83c9e300 2019-12-07T19:34:04.648+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx 1:41fe8a11:::smithi13140796-45:head [read 0~753664 out=753664b,checksum in=4b out=8b] ov 21'6 2019-12-07T19:34:04.648+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op 1:41fe8a11:::smithi13140796-45:head [read 0~753664 out=753664b,checksum in=4b out=8b] 2019-12-07T19:34:04.648+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op read 0~753664 out=753664b 2019-12-07T19:34:04.648+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op checksum in=4b out=8b 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx alloc reply 0x558e8417db80 result 0 2019-12-07T19:34:04.648+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:34:04.648+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] log_op_stats osd_op(client.4540.0:196 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~753664 out=753664b,checksum in=4b out=8b] snapc 0=[] ondisk+read+known_if_redirected e24) v8 inb 0 outb 753672 lat 2.994882 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(4)=3} 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 4 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 4 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 8 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.648+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 27:50 2019-12-07T19:34:04.648+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(196 smithi13140796-45 [read 0~753664 out=753664b,checksum out=8b] v0'0 uv6 ondisk = 0) v8 -- 0x558e8417db80 con 0x558e83d68000 2019-12-07T19:34:04.648+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e841771e0 finish 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.648+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e84177340 prio 127 cost 0 latency 0.007330 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.648+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=19, attrs_read=0)) v2 2019-12-07T19:34:04.648+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=19, attrs_read=0) 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=5 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=1 2019-12-07T19:34:04.648+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply Complete: ReadOp(tid=19, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,753664,0], need={0(0)=[0,8],1(2)=[0,8],3(5)=[0,8],4(1)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 753664, [0(0),188416, 1(2),188416, 3(5),188416, 4(1),188416]))}, priority=127, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),3(5),4(1)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,3(5)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head}, in_progress=) 2019-12-07T19:34:04.650+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 81 ==== MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=55, reqid=client.4540.0:206, at_version=27'9, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 96798+0+0 (unknown 0 0 0) 0x558e811b8800 con 0x558e818d7000 2019-12-07T19:34:04.650+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84177600 prio 127 cost 0 latency 0.000040 epoch 27 MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=55, reqid=client.4540.0:206, at_version=27'9, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.650+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=55, reqid=client.4540.0:206, at_version=27'9, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=55, reqid=client.4540.0:206, at_version=27'9, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=55, reqid=client.4540.0:206, at_version=27'9, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 osd.0 27 dequeue_op 0x558e84177600 prio 127 cost 0 latency 0.000084 MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=55, reqid=client.4540.0:206, at_version=27'9, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.fs1( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=55, reqid=client.4540.0:206, at_version=27'9, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'8], crt=21'6) [27'9 (27'8) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:206 2019-12-07T19:34:01.662593+0000 0 ObjectCleanRegions clean_offsets: [0~966656,1343488~18446744073708208127], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'9 (0'0,27'9] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'8 crt=21'6 lcod 27'8 active+remapped mbc={} ps=[2~1]] add_log_entry 27'9 (27'8) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:206 2019-12-07T19:34:01.662593+0000 0 ObjectCleanRegions clean_offsets: [0~966656,1343488~18446744073708208127], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'9 (0'0,27'9] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'8 crt=21'6 lcod 27'8 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 9 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'9 (0'0,27'9] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'8 crt=21'6 lcod 27'8 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.650+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e8106c300 seq 25 2019-12-07T19:34:04.650+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 2 keys 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 2 keys = 0 2019-12-07T19:34:04.650+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:04.650+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3b000~17000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 1#1:f7919c75:::smithi13140796-20:head# 0x3b000~17000 - have 0x3b000 (241664) bytes fadvise_flags 0x0 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x3b000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8106c300 12 blobs 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 fbmap_alloc 0x558e80233300 allocate 0x17000/1000,17000,0 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x336000~17000/1000,17000,0 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x336000~17000] 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b28850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f956c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f956c0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83b28850) get_ref 0x0~2000 Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f956c0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83b28850) get_ref init 0x2000, 1000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3b000~2000: 0x0~2000 Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f956c0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83f956d8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b28690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b289a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b289a0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83b28690) get_ref 0x0~2000 Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b289a0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83b28690) get_ref init 0x2000, 1000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3d000~2000: 0x0~2000 Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b289a0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83b289b8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b28a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b281c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b281c0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83b28a80) get_ref 0x0~2000 Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b281c0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83b28a80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3f000~2000: 0x0~2000 Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b281c0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83b281d8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b287e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b28150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b28150 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83b287e0) get_ref 0x0~2000 Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b28150 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83b287e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x41000~2000: 0x0~2000 Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28150 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83b28168 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b28770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b280e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b280e0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83b28770) get_ref 0x0~2000 Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b280e0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83b28770) get_ref init 0x2000, 1000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x43000~2000: 0x0~2000 Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b280e0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83b280f8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b28700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b28070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b28070 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83b28700) get_ref 0x0~2000 Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b28070 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83b28700) get_ref init 0x2000, 1000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x45000~2000: 0x0~2000 Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28070 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83b28088 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f48c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f48af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48af0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83f48c40) get_ref 0x0~2000 Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f48af0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83f48c40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x47000~2000: 0x0~2000 Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48af0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83f48b08 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f48d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03f10 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83f48d90) get_ref 0x0~2000 Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03f10 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83f48d90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x49000~2000: 0x0~2000 Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03f10 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d03f28 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d02b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02b60 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83d03ea0) get_ref 0x0~2000 Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02b60 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83d03ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4b000~2000: 0x0~2000 Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02b60 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d02b78 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d03e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f340 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83d03e30) get_ref 0x0~2000 Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f340 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83d03e30) get_ref init 0x2000, 1000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4d000~2000: 0x0~2000 Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f340 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0f358 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0f260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f2d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f2d0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0f260) get_ref 0x0~2000 Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f2d0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0f260) get_ref init 0x2000, 1000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4f000~2000: 0x0~2000 Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0f2e8 in 0x558e8102e000) _discard 0x0~2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0f960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af8d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0f960 blob([!~1000,0x34c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8d20 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0f960) get_ref 0x1000~1000 Blob(0x558e83d0f960 blob([!~1000,0x34c000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8d20 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.blob(0x558e83d0f960) get_ref init 0x2000, 1000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x51000~1000: 0x1000~1000 Blob(0x558e83d0f960 blob([!~1000,0x34c000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558e83af8d20 sbid 0x0)) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83af8d38 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x52000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83c72608) dirty_range mark inline shard dirty 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3b000~17000 = 0 2019-12-07T19:34:04.650+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.650+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.0000000027.00000000000000000009' <- 0000000027.00000000000000000009 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c300 cost 149238 (13 ios * 4000 + 97238 bytes) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c300 onodes 0x558e83c72500 shared_blobs 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83c72608) update 1#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83c72608) update inline shard 892 bytes from 42 extents 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:f7919c75:::smithi13140796-20:head# is 1455 (557 bytes onode + 2 bytes spanning blobs + 896 bytes inline extents) 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c300 allocated 0x[336000~17000] released 0x[] 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 freelist allocate 0x336000~17000 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 prepare 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8106c300 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'9 (0'0,27'9] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'8 crt=21'6 lcod 27'8 active+remapped mbc={} ps=[2~1]] op_applied version 27'9 2019-12-07T19:34:04.650+0000 7fe2c8267700 10 osd.0 27 dequeue_op 0x558e84177600 finish 2019-12-07T19:34:04.650+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.650+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 aio_wait 2019-12-07T19:34:04.650+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c300 2019-12-07T19:34:04.650+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 io_done 2019-12-07T19:34:04.650+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.650+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.650+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] objects_read_async_cb: got: {1:41fe8a11:::smithi13140796-45:head=0,{0~753664(753664)}} 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] objects_read_async_cb: cache: ExtentCache( ) 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] finish_checksum 2019-12-07T19:34:04.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3575 < min_flush_size 524288 2019-12-07T19:34:04.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72500 had 1 2019-12-07T19:34:04.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:04.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3598 < min_flush_size 524288 2019-12-07T19:34:04.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3598 < min_flush_size 524288 2019-12-07T19:34:04.651+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.651+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc4493~e0e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.651+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc4493 0xc4493~e0e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.651+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.651+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x94493 2019-12-07T19:34:04.651+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x493 2019-12-07T19:34:04.651+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.651+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x2a1 and padding block with 0xd5f 2019-12-07T19:34:04.651+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx 0x558e83c49800 2019-12-07T19:34:04.651+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx 1:41fe8a11:::smithi13140796-45:head [read 0~753664 out=753664b,checksum in=4b out=8b] ov 21'6 2019-12-07T19:34:04.651+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op 1:41fe8a11:::smithi13140796-45:head [read 0~753664 out=753664b,checksum in=4b out=8b] 2019-12-07T19:34:04.651+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op read 0~753664 out=753664b 2019-12-07T19:34:04.651+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op checksum in=4b out=8b 2019-12-07T19:34:04.651+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc52a1 2019-12-07T19:34:04.651+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.651+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.651+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.651+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] execute_ctx alloc reply 0x558e81a03680 result 0 2019-12-07T19:34:04.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000526329s (0.000377565s flush + 0.000148764s kv commit) 2019-12-07T19:34:04.651+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:34:04.651+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8106c300> 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 kv_submitted 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c300 2019-12-07T19:34:04.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 finishing 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c300 onodes 0x558e83c72500 2019-12-07T19:34:04.651+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] log_op_stats osd_op(client.4540.0:197 1.2s0 1:41fe8a11:::smithi13140796-45:head [read 0~753664 out=753664b,checksum in=4b out=8b] snapc 0=[] ondisk+read+known_if_redirected e24) v8 inb 0 outb 753672 lat 2.997776 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af8d38 in 0x558e8102e000) _finish_write discard buffer(0x558e841ed8c0 space 0x558e83af8d38 0x1000~1000 writing nocache) 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b28088 in 0x558e8102e000) _finish_write discard buffer(0x558e83d10510 space 0x558e83b28088 0x0~2000 writing nocache) 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b280f8 in 0x558e8102e000) _finish_write discard buffer(0x558e83d10360 space 0x558e83b280f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b28168 in 0x558e8102e000) _finish_write discard buffer(0x558e83d11320 space 0x558e83b28168 0x0~2000 writing nocache) 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b281d8 in 0x558e8102e000) _finish_write discard buffer(0x558e83d11440 space 0x558e83b281d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b289b8 in 0x558e8102e000) _finish_write discard buffer(0x558e80229ef0 space 0x558e83b289b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d02b78 in 0x558e8102e000) _finish_write discard buffer(0x558e841eccf0 space 0x558e83d02b78 0x0~2000 writing nocache) 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03f28 in 0x558e8102e000) _finish_write discard buffer(0x558e841ecd80 space 0x558e83d03f28 0x0~2000 writing nocache) 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0f2e8 in 0x558e8102e000) _finish_write discard buffer(0x558e841ec240 space 0x558e83d0f2e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0f358 in 0x558e8102e000) _finish_write discard buffer(0x558e841ecc60 space 0x558e83d0f358 0x0~2000 writing nocache) 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f48b08 in 0x558e8102e000) _finish_write discard buffer(0x558e841ece10 space 0x558e83f48b08 0x0~2000 writing nocache) 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f956d8 in 0x558e8102e000) _finish_write discard buffer(0x558e84032cf0 space 0x558e83f956d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c300 done 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:04.651+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c300 [] 2019-12-07T19:34:04.651+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.651+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(4)=3} 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 4 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 4 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 8 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.651+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.651+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 27/26 ECSubWriteReply(tid=55, last_complete=27'9, committed=1, applied=1)) v2 -- 0x558e8021f800 con 0x558e818d7000 2019-12-07T19:34:04.651+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 27:51 2019-12-07T19:34:04.651+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.651+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(197 smithi13140796-45 [read 0~753664 out=753664b,checksum out=8b] v0'0 uv6 ondisk = 0) v8 -- 0x558e81a03680 con 0x558e83d68000 2019-12-07T19:34:04.651+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e84177340 finish 2019-12-07T19:34:04.651+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.651+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 141 ==== MOSDECSubOpWrite(1.9s3 27/26 ECSubWrite(tid=105, reqid=client.4540.0:209, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2 ==== 1027+0+0 (unknown 0 0 0) 0x558e811b6000 con 0x558e818d7c00 2019-12-07T19:34:04.651+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84177760 prio 127 cost 0 latency 0.000022 epoch 27 MOSDECSubOpWrite(1.9s3 27/26 ECSubWrite(tid=105, reqid=client.4540.0:209, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2 2019-12-07T19:34:04.651+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 27/26 ECSubWrite(tid=105, reqid=client.4540.0:209, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.651+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.651+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 27/26 ECSubWrite(tid=105, reqid=client.4540.0:209, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 27/26 ECSubWrite(tid=105, reqid=client.4540.0:209, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2) prio 127 cost 0 e27) pg 0x558e8192b000 2019-12-07T19:34:04.652+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e84177760 prio 127 cost 0 latency 0.000066 MOSDECSubOpWrite(1.9s3 27/26 ECSubWrite(tid=105, reqid=client.4540.0:209, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2 pg pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.9s3 27/26 ECSubWrite(tid=105, reqid=client.4540.0:209, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2 2019-12-07T19:34:04.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'8], crt=21'6) [] 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'7 (21'3) clone 1:9694a0e5:::smithi13140796-3:4 by unknown.0.0:0 2019-12-07T19:33:58.718796+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'8 (21'3) modify 1:9694a0e5:::smithi13140796-3:head by client.4540.0:209 0.000000 0 ObjectCleanRegions clean_offsets: [589824~18446744073708961791], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 8 2019-12-07T19:34:04.652+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.652+0000 7fe2cb26d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.652+0000 7fe2cb26d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.652+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e811faf00 seq 21 2019-12-07T19:34:04.652+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head#8 onode 0x558e81a03b80 txc 0x558e811faf00 2019-12-07T19:34:04.652+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head#8 0x0 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore.blob(0x558e8194e690) put_ref 0x0~2000 Blob(0x558e8194e690 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=2)))) 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore.blob(0x558e8194f340) put_ref 0x0~1000 Blob(0x558e8194f340 blob([0x39000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=2)))) 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e81a03c88) dirty_range mark inline shard dirty 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e8194e690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=2)))) 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x37000~2000] 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1))) 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~1000: 0x0~1000 Blob(0x558e8194f340 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=2)))) 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x39000~1000] 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=1))) 2019-12-07T19:34:04.652+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove gen and maybe_unshared_blobs 0x558e817e8d20,0x558e8194f3b0 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove checking for unshareable blobs on 0x558e83b08280 3#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:04.652+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head#8 = 0 2019-12-07T19:34:04.652+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.652+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:04.652+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811faf00 cost 5562 (1 ios * 4000 + 1562 bytes) 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811faf00 onodes shared_blobs 0x558e817e8d20,0x558e8194f3b0 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2812 is 11 SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1))) 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2813 is 11 SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=1))) 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811faf00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.652+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 prepare 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811faf00 2019-12-07T19:34:04.652+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 io_done 2019-12-07T19:34:04.652+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e84177760 finish 2019-12-07T19:34:04.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1386 < min_flush_size 524288 2019-12-07T19:34:04.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:04.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03b80 had 1 2019-12-07T19:34:04.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1409 < min_flush_size 524288 2019-12-07T19:34:04.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1409 < min_flush_size 524288 2019-12-07T19:34:04.652+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.652+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc52a1~581 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.652+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc52a1 0xc52a1~581 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.652+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.652+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x952a1 2019-12-07T19:34:04.652+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x2a1 2019-12-07T19:34:04.652+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.652+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x822 and padding block with 0x7de 2019-12-07T19:34:04.652+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc5822 2019-12-07T19:34:04.652+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.652+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.652+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.652+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000132017s (0.000000113s flush + 0.000131904s kv commit) 2019-12-07T19:34:04.652+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811faf00> 2019-12-07T19:34:04.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 kv_submitted 2019-12-07T19:34:04.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811faf00 2019-12-07T19:34:04.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 finishing 2019-12-07T19:34:04.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811faf00 onodes 2019-12-07T19:34:04.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811faf00 done 2019-12-07T19:34:04.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:04.652+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811faf00 [] 2019-12-07T19:34:04.652+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.652+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.652+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.652+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 27/26 ECSubWriteReply(tid=105, last_complete=27'8, committed=1, applied=1)) v2 -- 0x558e84052e00 con 0x558e818d7c00 2019-12-07T19:34:04.652+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.653+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 82 ==== MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=56, reqid=client.4540.0:216, at_version=0'0, trim_to=0'0, roll_forward_to=27'7)) v2 ==== 1028+0+0 (unknown 0 0 0) 0x558e819d1800 con 0x558e818d7000 2019-12-07T19:34:04.653+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edc6e0 prio 127 cost 0 latency 0.000031 epoch 27 MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=56, reqid=client.4540.0:216, at_version=0'0, trim_to=0'0, roll_forward_to=27'7)) v2 2019-12-07T19:34:04.653+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=56, reqid=client.4540.0:216, at_version=0'0, trim_to=0'0, roll_forward_to=27'7)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=56, reqid=client.4540.0:216, at_version=0'0, trim_to=0'0, roll_forward_to=27'7)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=56, reqid=client.4540.0:216, at_version=0'0, trim_to=0'0, roll_forward_to=27'7)) v2) prio 127 cost 0 e27) pg 0x558e819e4000 2019-12-07T19:34:04.654+0000 7fe2c7265700 10 osd.0 27 dequeue_op 0x558e83edc6e0 prio 127 cost 0 latency 0.000219 MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=56, reqid=client.4540.0:216, at_version=0'0, trim_to=0'0, roll_forward_to=27'7)) v2 pg pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.654+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.1s1 27/26 ECSubWrite(tid=56, reqid=client.4540.0:216, at_version=0'0, trim_to=0'0, roll_forward_to=27'7)) v2 2019-12-07T19:34:04.654+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'3 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'7], crt=21'3) [] 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'4 (21'3) clone 1:891f0d13:::smithi13140796-14:4 by unknown.0.0:0 2019-12-07T19:33:58.819566+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'5 (21'3) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:214 2019-12-07T19:34:01.667796+0000 0 ObjectCleanRegions clean_offsets: [0~229376,884736~18446744073708666879], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'6 (27'5) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:215 2019-12-07T19:34:01.670014+0000 0 ObjectCleanRegions clean_offsets: [0~884736,1261568~18446744073708290047], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'7 (27'6) modify 1:891f0d13:::smithi13140796-14:head by client.4540.0:216 2019-12-07T19:34:01.670030+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.654+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 7 2019-12-07T19:34:04.654+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.654+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.654+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.654+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e811fa600 seq 21 2019-12-07T19:34:04.654+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.654+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:04.654+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa600 cost 5421 (1 ios * 4000 + 1421 bytes) 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa600 onodes shared_blobs 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.654+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 prepare 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa600 2019-12-07T19:34:04.654+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 io_done 2019-12-07T19:34:04.654+0000 7fe2c7265700 10 osd.0 27 dequeue_op 0x558e83edc6e0 finish 2019-12-07T19:34:04.654+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.654+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.654+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.654+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1232 < min_flush_size 524288 2019-12-07T19:34:04.654+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:04.654+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1255 < min_flush_size 524288 2019-12-07T19:34:04.654+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1255 < min_flush_size 524288 2019-12-07T19:34:04.654+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.654+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc5822~4e7 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.654+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc5822 0xc5822~4e7 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.654+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.654+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x95822 2019-12-07T19:34:04.654+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x822 2019-12-07T19:34:04.654+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.654+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd09 and padding block with 0x2f7 2019-12-07T19:34:04.654+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 142 ==== MOSDECSubOpRead(1.es4 27/26 ECSubRead(tid=107, to_read={1:733d7474:::smithi13140796-5:head=0,262144,0}, subchunks={1:733d7474:::smithi13140796-5:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 263+0+0 (unknown 0 0 0) 0x558e8417c280 con 0x558e818d7c00 2019-12-07T19:34:04.654+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edc840 prio 3 cost 0 latency 0.000021 epoch 27 MOSDECSubOpRead(1.es4 27/26 ECSubRead(tid=107, to_read={1:733d7474:::smithi13140796-5:head=0,262144,0}, subchunks={1:733d7474:::smithi13140796-5:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.654+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpRead(1.es4 27/26 ECSubRead(tid=107, to_read={1:733d7474:::smithi13140796-5:head=0,262144,0}, subchunks={1:733d7474:::smithi13140796-5:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.654+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 83 ==== MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=59, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8417d180 con 0x558e818d7000 2019-12-07T19:34:04.654+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84177600 prio 127 cost 0 latency 0.000013 epoch 27 MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=59, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.654+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=59, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.654+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.654+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpRead(1.es4 27/26 ECSubRead(tid=107, to_read={1:733d7474:::smithi13140796-5:head=0,262144,0}, subchunks={1:733d7474:::smithi13140796-5:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.654+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.654+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpRead(1.es4 27/26 ECSubRead(tid=107, to_read={1:733d7474:::smithi13140796-5:head=0,262144,0}, subchunks={1:733d7474:::smithi13140796-5:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e81a98000 2019-12-07T19:34:04.654+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 84 ==== MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=58, reqid=client.4540.0:207, at_version=27'10, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2556+0+0 (unknown 0 0 0) 0x558e83e56000 con 0x558e818d7000 2019-12-07T19:34:04.654+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84176dc0 prio 127 cost 0 latency 0.000031 epoch 27 MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=58, reqid=client.4540.0:207, at_version=27'10, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.654+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=58, reqid=client.4540.0:207, at_version=27'10, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.654+0000 7fe2c8a68700 10 osd.0 27 dequeue_op 0x558e83edc840 prio 3 cost 0 latency 0.000087 MOSDECSubOpRead(1.es4 27/26 ECSubRead(tid=107, to_read={1:733d7474:::smithi13140796-5:head=0,262144,0}, subchunks={1:733d7474:::smithi13140796-5:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.654+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.es4 27/26 ECSubRead(tid=107, to_read={1:733d7474:::smithi13140796-5:head=0,262144,0}, subchunks={1:733d7474:::smithi13140796-5:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.654+0000 7fe2c8a68700 25 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.654+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x200~200 2019-12-07T19:34:04.654+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x200~200 size 0x5000 (20480) 2019-12-07T19:34:04.654+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.654+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9180 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.654+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9180 sbid 0x0)) need {<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.654+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.654+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.654+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.654+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=59, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.654+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.654+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=59, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) pg 0x558e819e4000 2019-12-07T19:34:04.654+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e84177600 prio 127 cost 0 latency 0.000232 MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=59, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.654+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=59, to_read={1:891f0d13:::smithi13140796-14:head=0,4096,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.654+0000 7fe2cb26d700 25 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.654+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x0~1000 2019-12-07T19:34:04.654+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x4d000 (315392) 2019-12-07T19:34:04.654+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.654+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65730 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:04.654+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x0~1000 = 4096 2019-12-07T19:34:04.654+0000 7fe2cb26d700 20 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:34:04.654+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:04.654+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:04.654+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:34:04.654+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 315392 2019-12-07T19:34:04.654+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key 2019-12-07T19:34:04.654+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key = 0 2019-12-07T19:34:04.654+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.1s0 27/26 ECSubReadReply(tid=59, attrs_read=0)) v2 -- 0x558e8402e500 con 0x558e818d7000 2019-12-07T19:34:04.654+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e84177600 finish 2019-12-07T19:34:04.654+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9180 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e817e9198 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 _add level 1 near 0 on buffer(0x558e84033d40 space 0x558e817e9198 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x200~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x600~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x600~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9180 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x600~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xa00~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa00~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9180 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xa00~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xe00~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe00~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9180 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xe00~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1200~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1200~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9180 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9180 sbid 0x0)) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc5d09 2019-12-07T19:34:04.655+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.655+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.655+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000728570s (0.000000100s flush + 0.000728470s kv commit) 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa600> 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 kv_submitted 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa600 2019-12-07T19:34:04.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 finishing 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa600 onodes 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa600 done 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:04.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa600 [] 2019-12-07T19:34:04.655+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.655+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.655+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 27/26 ECSubWriteReply(tid=56, last_complete=27'7, committed=1, applied=1)) v2 -- 0x558e84053400 con 0x558e818d7000 2019-12-07T19:34:04.655+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=58, reqid=client.4540.0:207, at_version=27'10, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=58, reqid=client.4540.0:207, at_version=27'10, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e84176dc0 prio 127 cost 0 latency 0.000698 MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=58, reqid=client.4540.0:207, at_version=27'10, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.fs1( v 27'9 (0'0,27'9] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'9 (0'0,27'9] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=58, reqid=client.4540.0:207, at_version=27'10, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'9 (0'0,27'9] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'9], crt=21'6) [27'10 (27'9) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:207 2019-12-07T19:34:01.662612+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'9 crt=21'6 lcod 27'9 active+remapped mbc={} ps=[2~1]] add_log_entry 27'10 (27'9) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:207 2019-12-07T19:34:01.662612+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'9 crt=21'6 lcod 27'9 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 10 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'9 crt=21'6 lcod 27'9 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.655+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e8417a600 seq 26 2019-12-07T19:34:04.655+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 3 keys 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9180 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e817e9198 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 3 keys = 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 _add level 1 near 0 on buffer(0x558e841edb90 space 0x558e817e9198 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.655+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1200~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1600~200 2019-12-07T19:34:04.655+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1600~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.0000000027.00000000000000000010' <- 0000000027.00000000000000000010 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9180 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._fastinfo' <- _fastinfo 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1600~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1a00~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9180 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417a600 cost 6088 (1 ios * 4000 + 2088 bytes) 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417a600 onodes 0x558e83c72500 shared_blobs 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1a00~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 bluestore.extentmap(0x558e83c72608) update 1#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1e00~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:f7919c75:::smithi13140796-20:head# is 1455 (557 bytes onode + 2 bytes spanning blobs + 896 bytes inline extents) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8fc0 blob([0x3a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9180 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1e00~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417a600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2200~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2200~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ea10 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 prepare 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417a600 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ea10 sbid 0x0)) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 io_done 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'9 crt=21'6 lcod 27'9 active+remapped mbc={} ps=[2~1]] op_applied version 27'10 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.655+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e84176dc0 finish 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.655+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2627 < min_flush_size 524288 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72500 had 1 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:04.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2650 < min_flush_size 524288 2019-12-07T19:34:04.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2650 < min_flush_size 524288 2019-12-07T19:34:04.655+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc5d09~a5a to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.655+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc5d09 0xc5d09~a5a to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x95d09 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd09 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x763 and padding block with 0x89d 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ea10 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e8194ea28 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 _add level 1 near 0 on buffer(0x558e841edc20 space 0x558e8194ea28 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2200~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2600~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2600~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ea10 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2600~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2a00~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ea10 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2a00~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2e00~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ea10 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2e00~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3200~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3200~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ea10 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ea10 sbid 0x0)) need {<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc6763 2019-12-07T19:34:04.655+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.655+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.655+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000170028s (0.000000090s flush + 0.000169938s kv commit) 2019-12-07T19:34:04.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417a600> 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 kv_submitted 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417a600 2019-12-07T19:34:04.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 finishing 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417a600 onodes 0x558e83c72500 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a600 done 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:04.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417a600 [] 2019-12-07T19:34:04.655+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.655+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 349776 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ea10 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e8194ea28 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 _add level 1 near 0 on buffer(0x558e841edcb0 space 0x558e8194ea28 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3200~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3600~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3600~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ea10 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3600~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3a00~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ea10 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3a00~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3e00~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e930 blob([0x3c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ea10 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3e00~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x4200~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4200~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194ea80 blob([0x3e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194eb60 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194ea80 blob([0x3e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194eb60 sbid 0x0)) need {<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194ea80 blob([0x3e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194eb60 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore.BufferSpace(0x558e8194eb78 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 _add level 1 near 0 on buffer(0x558e841edd40 space 0x558e8194eb78 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x4200~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x4600~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4600~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194ea80 blob([0x3e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194eb60 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x4600~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x4a00~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194ea80 blob([0x3e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194eb60 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x4a00~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x4e00~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194ea80 blob([0x3e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194eb60 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x4e00~200 = 512 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x5200~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5200~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x5200~200 = 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x5600~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5600~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x5600~200 = 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x5a00~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x5a00~200 = 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x5e00~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x5e00~200 = 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x6200~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6200~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x6200~200 = 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x6600~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6600~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x6600~200 = 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x6a00~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x6a00~200 = 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x6e00~200 2019-12-07T19:34:04.655+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.655+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x6e00~200 = 0 2019-12-07T19:34:04.655+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x7200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x7200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x7600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x7600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x7a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x7a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x7e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x7e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x8200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x8200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x8600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x8600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x8a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x8a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x8e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x8e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x9200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x9200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x9600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x9600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x9a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x9a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x9e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x9e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xa200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xa200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xa600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xa600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xaa00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xaa00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xaa00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xae00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xae00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xae00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xb200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xb200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xb600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xb600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xba00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xba00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xba00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xbe00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xbe00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xbe00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xc200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xc200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xc600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xc600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xca00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xca00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xca00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xce00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xce00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xce00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xd200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xd200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xd600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xd600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xda00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xda00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xda00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xde00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xde00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xde00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xe200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xe200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xe600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xe600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xea00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xea00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xea00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xee00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xee00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xee00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xf200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xf200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xf600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xf600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xfa00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfa00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xfa00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xfe00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfe00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0xfe00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x10200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x10200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x10600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x10600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x10a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x10a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x10e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x10e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x11200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x11200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x11600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x11600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x11a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x11a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x11e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x11e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x12200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x12200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x12600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x12600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x12a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x12a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x12e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x12e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x13200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x13200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x13600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x13600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x13a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x13a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x13e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x13e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x14200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x14200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x14600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x14600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x14a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x14a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x14e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x14e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x15200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x15200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x15600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x15600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x15a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x15a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x15e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x15e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x16200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x16200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x16600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x16600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x16a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x16a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x16e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x16e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x17200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x17200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x17600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x17600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x17a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x17a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x17e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x17e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x18200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x18200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x18600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x18600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x18a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x18a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x18e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x18e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x19200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x19200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x19600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x19600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x19a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x19a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x19e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x19e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1a200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1a200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1a600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1a600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1aa00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1aa00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1aa00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1ae00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ae00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1ae00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1b200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1b200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1b600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1b600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1ba00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ba00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1ba00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1be00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1be00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1be00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1c200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1c200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1c600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1c600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1ca00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ca00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1ca00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1ce00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ce00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1ce00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1d200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1d200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1d600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1d600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1da00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1da00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1da00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1de00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1de00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1de00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1e200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1e200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1e600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1e600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1ea00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ea00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1ea00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1ee00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ee00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1ee00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1f200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1f200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1f600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1f600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1fa00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fa00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1fa00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1fe00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fe00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x1fe00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x20200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x20200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x20600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x20600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x20a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x20a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x20e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x20e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x21200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x21200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x21600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x21600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x21a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x21a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x21e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x21e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x22200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x22200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x22600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x22600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x22a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x22a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x22e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x22e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x23200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x23200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x23600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x23600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x23a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x23a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x23e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x23e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x24200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x24200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x24600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x24600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x24a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x24a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x24e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x24e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x25200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x25200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x25600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x25600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x25a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x25a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x25e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x25e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x26200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x26200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x26600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x26600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x26a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x26a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x26e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x26e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x27200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x27200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x27600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x27600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x27a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x27a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x27e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x27e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x28200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x28200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x28600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x28600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x28a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x28a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x28e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x28e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x29200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x29200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x29600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x29600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x29a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x29a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x29e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x29e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2a200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2a200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2a600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2a600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2aa00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2aa00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2aa00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2ae00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ae00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2ae00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2b200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2b200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2b600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2b600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2ba00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ba00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2ba00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2be00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2be00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2be00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2c200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2c200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2c600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2c600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2ca00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ca00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2ca00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2ce00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ce00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2ce00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2d200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2d200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2d600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2d600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2da00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2da00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2da00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2de00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2de00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2de00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2e200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2e200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2e600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2e600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2ea00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ea00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2ea00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2ee00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ee00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2ee00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2f200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2f200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2f600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2f600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2fa00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fa00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2fa00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2fe00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fe00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x2fe00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x30200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x30200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x30600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x30600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x30a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x30a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x30e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x30e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x31200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x31200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x31600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x31600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x31a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x31a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x31e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x31e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x32200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x32200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x32600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x32600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x32a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x32a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x32e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x32e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x33200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x33200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x33600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x33600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x33a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x33a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x33e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x33e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x34200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x34200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x34600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x34600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x34a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x34a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x34e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x34e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x35200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x35200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x35600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x35600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x35a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x35a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x35e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x35e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x36200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x36200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x36600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x36600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x36a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x36a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x36e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x36e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x37200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x37200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x37600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x37600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x37a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x37a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x37e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x37e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x38200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x38200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x38600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x38600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x38a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x38a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x38e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x38e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x39200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x39200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x39600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x39600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x39a00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39a00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x39a00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x39e00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39e00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x39e00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3a200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3a200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3a600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3a600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3aa00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3aa00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3aa00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3ae00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ae00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3ae00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3b200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3b200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3b600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3b600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3ba00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ba00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3ba00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3be00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3be00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3be00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3c200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3c200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3c600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3c600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3ca00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ca00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3ca00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3ce00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ce00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3ce00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3d200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3d200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3d600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3d600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3da00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3da00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3da00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3de00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3de00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3de00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3e200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3e200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3e600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3e600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3ea00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ea00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3ea00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3ee00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ee00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3ee00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3f200~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f200~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3f200~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3f600~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f600~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3f600~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3fa00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fa00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3fa00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3fe00~200 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fe00~200 size 0x5000 (20480) 2019-12-07T19:34:04.656+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 0x3fe00~200 = 0 2019-12-07T19:34:04.656+0000 7fe2c8a68700 20 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=10240 2019-12-07T19:34:04.657+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:733d7474:::smithi13140796-5:head 2019-12-07T19:34:04.657+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:733d7474:::smithi13140796-5:head 2019-12-07T19:34:04.657+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.es4_head 4#1:733d7474:::smithi13140796-5:head# 2019-12-07T19:34:04.657+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 20480 2019-12-07T19:34:04.657+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key 2019-12-07T19:34:04.657+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.es4_head 4#1:733d7474:::smithi13140796-5:head# hinfo_key = 0 2019-12-07T19:34:04.657+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.es0 27/26 ECSubReadReply(tid=107, attrs_read=0)) v2 -- 0x558e811dc780 con 0x558e818d7c00 2019-12-07T19:34:04.657+0000 7fe2c8a68700 10 osd.0 27 dequeue_op 0x558e83edc840 finish 2019-12-07T19:34:04.657+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.657+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 143 ==== MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=108, reqid=client.4540.0:191, at_version=0'0, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2 ==== 1028+0+0 (unknown 0 0 0) 0x558e83e55800 con 0x558e818d7c00 2019-12-07T19:34:04.657+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e841774a0 prio 127 cost 0 latency 0.000024 epoch 27 MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=108, reqid=client.4540.0:191, at_version=0'0, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2 2019-12-07T19:34:04.657+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=108, reqid=client.4540.0:191, at_version=0'0, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.657+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.657+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=108, reqid=client.4540.0:191, at_version=0'0, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.657+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.657+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=108, reqid=client.4540.0:191, at_version=0'0, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) pg 0x558e841a4000 2019-12-07T19:34:04.657+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e841774a0 prio 127 cost 0 latency 0.000089 MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=108, reqid=client.4540.0:191, at_version=0'0, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2 pg pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=7 mbc={} ps=[2~1]] 2019-12-07T19:34:04.657+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=7 mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=108, reqid=client.4540.0:191, at_version=0'0, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2 2019-12-07T19:34:04.657+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=7 mbc={} ps=[2~1]] append_log log((0'0,27'18], crt=27'18) [] 2019-12-07T19:34:04.657+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=7 mbc={} ps=[2~1]] append_log approx pg log length = 18 2019-12-07T19:34:04.657+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=7 mbc={} ps=[2~1]] append_log transaction_applied = 0 2019-12-07T19:34:04.657+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=7 mbc={} ps=[2~1]] append_log 0(5) is async_recovery or backfill target 2019-12-07T19:34:04.657+0000 7fe2c7a66700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.657+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:04.657+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.657+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e8195e300 seq 7 2019-12-07T19:34:04.657+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.657+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.657+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.657+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e300 cost 5342 (1 ios * 4000 + 1342 bytes) 2019-12-07T19:34:04.657+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e300 onodes shared_blobs 2019-12-07T19:34:04.657+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e300 allocated 0x[] released 0x[] 2019-12-07T19:34:04.657+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 prepare 2019-12-07T19:34:04.657+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e300 2019-12-07T19:34:04.657+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 io_done 2019-12-07T19:34:04.657+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e841774a0 finish 2019-12-07T19:34:04.657+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:04.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1164 < min_flush_size 524288 2019-12-07T19:34:04.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1164 < min_flush_size 524288 2019-12-07T19:34:04.657+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc6763~48c to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.657+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc6763 0xc6763~48c to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.657+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.657+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x96763 2019-12-07T19:34:04.657+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x763 2019-12-07T19:34:04.657+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.657+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xbef and padding block with 0x411 2019-12-07T19:34:04.657+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc6bef 2019-12-07T19:34:04.657+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.657+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.657+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.657+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000116862s (0.000000095s flush + 0.000116767s kv commit) 2019-12-07T19:34:04.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e300> 2019-12-07T19:34:04.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.657+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 kv_submitted 2019-12-07T19:34:04.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e300 2019-12-07T19:34:04.657+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 finishing 2019-12-07T19:34:04.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e300 onodes 2019-12-07T19:34:04.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 done 2019-12-07T19:34:04.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.657+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e300 [] 2019-12-07T19:34:04.657+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.657+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.657+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 27/26 ECSubWriteReply(tid=108, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e84053e00 con 0x558e818d7c00 2019-12-07T19:34:04.657+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.657+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.659+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 27/26 ECSubWriteReply(tid=58, last_complete=27'10, committed=1, applied=1)) v2 -- 0x558e818a8c00 con 0x558e818d7000 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.659+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 144 ==== pg_update_log_missing(1.7s3 epoch 27/26 rep_tid 110 entries 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 ==== 258+0+0 (unknown 0 0 0) 0x558e83c666c0 con 0x558e818d7c00 2019-12-07T19:34:04.659+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84177080 prio 127 cost 0 latency 0.000025 epoch 27 pg_update_log_missing(1.7s3 epoch 27/26 rep_tid 110 entries 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 2019-12-07T19:34:04.659+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=pg_update_log_missing(1.7s3 epoch 27/26 rep_tid 110 entries 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=pg_update_log_missing(1.7s3 epoch 27/26 rep_tid 110 entries 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=pg_update_log_missing(1.7s3 epoch 27/26 rep_tid 110 entries 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e27) pg 0x558e8122e000 2019-12-07T19:34:04.659+0000 7fe2c8267700 10 osd.0 27 dequeue_op 0x558e84177080 prio 127 cost 0 latency 0.000382 pg_update_log_missing(1.7s3 epoch 27/26 rep_tid 110 entries 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 pg pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.659+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] _handle_message: pg_update_log_missing(1.7s3 epoch 27/26 rep_tid 110 entries 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 osd.0 pg_epoch: 27 pg[1.7s3( v 21'9 (0'0,21'9] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] do_update_log_missing op_trim_to = -- op_roll_forward_to = -- 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 update missing, append 27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active+remapped mbc={} ps=[2~1]] append_log_entries_update_missing trim_to bool = 0 trim_to = 0'0 2019-12-07T19:34:04.659+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.659+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e8417ac00 seq 29 2019-12-07T19:34:04.659+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.0000000027.00000000000000000010' <- 0000000027.00000000000000000010 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._fastinfo' <- _fastinfo 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.659+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:04.659+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417ac00 cost 4717 (1 ios * 4000 + 717 bytes) 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417ac00 onodes shared_blobs 2019-12-07T19:34:04.659+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.660+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 prepare 2019-12-07T19:34:04.660+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417ac00 2019-12-07T19:34:04.660+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 io_done 2019-12-07T19:34:04.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.660+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 lua=21'9 crt=21'9 lcod 21'9 active+remapped mbc={} ps=[2~1]] op_applied version 27'10 2019-12-07T19:34:04.660+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 567 < min_flush_size 524288 2019-12-07T19:34:04.660+0000 7fe2c8267700 10 osd.0 27 dequeue_op 0x558e84177080 finish 2019-12-07T19:34:04.660+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:04.660+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 590 < min_flush_size 524288 2019-12-07T19:34:04.660+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 590 < min_flush_size 524288 2019-12-07T19:34:04.660+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.660+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc6bef~24e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.660+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc6bef 0xc6bef~24e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.660+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.660+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x96bef 2019-12-07T19:34:04.660+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xbef 2019-12-07T19:34:04.660+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.660+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xe3d and padding block with 0x1c3 2019-12-07T19:34:04.660+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc6e3d 2019-12-07T19:34:04.660+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.660+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.660+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.660+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000099859s (0.000000101s flush + 0.000099758s kv commit) 2019-12-07T19:34:04.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417ac00> 2019-12-07T19:34:04.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.660+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 kv_submitted 2019-12-07T19:34:04.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417ac00 2019-12-07T19:34:04.660+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 finishing 2019-12-07T19:34:04.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417ac00 onodes 2019-12-07T19:34:04.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417ac00 done 2019-12-07T19:34:04.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:04.660+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417ac00 [] 2019-12-07T19:34:04.660+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.660+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.660+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.660+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_update_log_missing_reply(1.7s0 epoch 27/26 rep_tid 110 lcod 27'10) v3 -- 0x558e84065400 con 0x558e818d7c00 2019-12-07T19:34:04.660+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.662+0000 7fe2e4535700 10 osd.0 27 tick 2019-12-07T19:34:04.662+0000 7fe2e4535700 10 osd.0 27 do_waiters -- start 2019-12-07T19:34:04.662+0000 7fe2e4535700 10 osd.0 27 do_waiters -- finish 2019-12-07T19:34:04.662+0000 7fe2e4535700 20 osd.0 27 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:04.662+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 145 ==== MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=111, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e811dc780 con 0x558e818d7c00 2019-12-07T19:34:04.662+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e819434a0 prio 3 cost 0 latency 0.000023 epoch 27 MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=111, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.662+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=111, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=111, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=111, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e8122e000 2019-12-07T19:34:04.662+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e819434a0 prio 3 cost 0 latency 0.000101 MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=111, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.662+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=111, to_read={1:ecc72bdb:::smithi13140796-10:head=0,262144,0}, subchunks={1:ecc72bdb:::smithi13140796-10:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.662+0000 7fe2c425f700 25 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.662+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# 0x0~40000 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~40000 size 0xa000 (40960) 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81867ab0 blob([0x45000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9260 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866700 blob([0x47000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9030 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866930 blob([0x49000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d00e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64850 blob([0x4b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c647e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64770 blob([0x4d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64700 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81866700 blob([0x47000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9030 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81866930 blob([0x49000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d00e0 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81867ab0 blob([0x45000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9260 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c64770 blob([0x4d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64700 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c64850 blob([0x4b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c647e0 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81866700 blob([0x47000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9030 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e817e9048 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e841ed950 space 0x558e817e9048 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81866930 blob([0x49000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d00e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e811d00f8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e841ec870 space 0x558e811d00f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81867ab0 blob([0x45000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9260 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e817e9278 in 0x558e80219880) _discard 0x1000~1000 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e841ec900 space 0x558e817e9278 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c64770 blob([0x4d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64700 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83c64718 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e841ec990 space 0x558e83c64718 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c64850 blob([0x4b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c647e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83c647f8 in 0x558e80219880) _discard 0x0~2000 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e841eca20 space 0x558e83c647f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.662+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# 0x0~40000 = 40960 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=40960 len=40960 2019-12-07T19:34:04.662+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:ecc72bdb:::smithi13140796-10:head 2019-12-07T19:34:04.662+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:ecc72bdb:::smithi13140796-10:head 2019-12-07T19:34:04.662+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# 2019-12-07T19:34:04.662+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 40960 2019-12-07T19:34:04.662+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# hinfo_key 2019-12-07T19:34:04.662+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ecc72bdb:::smithi13140796-10:head# hinfo_key = 0 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:ecc72bdb:::smithi13140796-10:head 2019-12-07T19:34:04.662+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 27/26 ECSubReadReply(tid=111, attrs_read=0)) v2 -- 0x558e8417cc80 con 0x558e818d7c00 2019-12-07T19:34:04.662+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e819434a0 finish 2019-12-07T19:34:04.662+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.663+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 30 ==== MOSDECSubOpWrite(1.5s4 27/26 ECSubWrite(tid=34, reqid=client.4540.0:203, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 178364+0+0 (unknown 0 0 0) 0x558e84040800 con 0x558e818d6000 2019-12-07T19:34:04.663+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e84177e40 prio 127 cost 0 latency 0.000036 epoch 27 MOSDECSubOpWrite(1.5s4 27/26 ECSubWrite(tid=34, reqid=client.4540.0:203, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.663+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGOpItem(op=MOSDECSubOpWrite(1.5s4 27/26 ECSubWrite(tid=34, reqid=client.4540.0:203, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDECSubOpWrite(1.5s4 27/26 ECSubWrite(tid=34, reqid=client.4540.0:203, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDECSubOpWrite(1.5s4 27/26 ECSubWrite(tid=34, reqid=client.4540.0:203, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e27) pg 0x558e83d07000 2019-12-07T19:34:04.663+0000 7fe2c9269700 10 osd.0 27 dequeue_op 0x558e84177e40 prio 127 cost 0 latency 0.000089 MOSDECSubOpWrite(1.5s4 27/26 ECSubWrite(tid=34, reqid=client.4540.0:203, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 active m=2 mbc={} ps=[2~1]] 2019-12-07T19:34:04.663+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 active m=2 mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.5s4 27/26 ECSubWrite(tid=34, reqid=client.4540.0:203, at_version=27'8, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:04.663+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 23'7 lc 21'1 (0'0,23'7] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 active m=2 mbc={} ps=[2~1]] append_log log((0'0,23'7], crt=23'7) [27'8 (0'0) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:203 0.000000 0 ObjectCleanRegions clean_offsets: [704512~18446744073708847103], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.663+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'7 crt=23'7 active m=2 mbc={} ps=[2~1]] add_log_entry 27'8 (0'0) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:203 0.000000 0 ObjectCleanRegions clean_offsets: [704512~18446744073708847103], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.663+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'7 crt=23'7 active m=2 mbc={} ps=[2~1]] append_log approx pg log length = 8 2019-12-07T19:34:04.663+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'7 crt=23'7 active m=2 mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.663+0000 7fe2c9269700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.663+0000 7fe2c9269700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.663+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e83c56c00 seq 6 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.5s4_head 0x558e8403fe00) get_onode oid 4#1:a7686634:::smithi13140796-35:head# key 0x848000000000000001a76866'4!smithi13140796-35!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.663+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1127 2019-12-07T19:34:04.663+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# = 0 2019-12-07T19:34:04.663+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# 3 keys 2019-12-07T19:34:04.663+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# 3 keys = 0 2019-12-07T19:34:04.663+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:04.663+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:04.663+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# 0x0~2b000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 4#1:a7686634:::smithi13140796-35:head# 0x0~2b000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.663+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~2b000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83c56c00 22 blobs 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.663+0000 7fe2c9269700 10 fbmap_alloc 0x558e80233300 allocate 0x2b000/1000,2b000,0 2019-12-07T19:34:04.663+0000 7fe2c9269700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x34d000~2b000/1000,2b000,0 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x34d000~2b000] 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6f7a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841db7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6f7a0 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db7a0 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6f7a0) get_ref 0x0~2000 Blob(0x558e83e6f7a0 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db7a0 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6f7a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83e6f7a0 blob([0x34d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db7a0 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e841db7b8 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e811d0380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e811d0380 blob([0x34f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f730 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.blob(0x558e811d0380) get_ref 0x0~2000 Blob(0x558e811d0380 blob([0x34f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f730 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.blob(0x558e811d0380) get_ref init 0x2000, 1000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e811d0380 blob([0x34f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6f730 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83e6f748 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6f5e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6f5e0 blob([0x351000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f570 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6f5e0) get_ref 0x0~2000 Blob(0x558e83e6f5e0 blob([0x351000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f570 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6f5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83e6f5e0 blob([0x351000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6f570 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83e6f588 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6f500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6f500 blob([0x353000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f490 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6f500) get_ref 0x0~2000 Blob(0x558e83e6f500 blob([0x353000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f490 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6f500) get_ref init 0x2000, 1000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83e6f500 blob([0x353000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6f490 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83e6f4a8 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6f260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6f260 blob([0x355000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f340 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6f260) get_ref 0x0~2000 Blob(0x558e83e6f260 blob([0x355000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f340 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.blob(0x558e83e6f260) get_ref init 0x2000, 1000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83e6f260 blob([0x355000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6f340 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83e6f358 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841dbc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841dbce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841dbc70 blob([0x357000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbce0 sbid 0x0)) 2019-12-07T19:34:04.663+0000 7fe2c9269700 20 bluestore.blob(0x558e841dbc70) get_ref 0x0~2000 Blob(0x558e841dbc70 blob([0x357000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbce0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e841dbc70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e841dbc70 blob([0x357000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbce0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e841dbcf8 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841dbd50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841dbdc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841dbd50 blob([0x359000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbdc0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e841dbd50) get_ref 0x0~2000 Blob(0x558e841dbd50 blob([0x359000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbdc0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e841dbd50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e841dbd50 blob([0x359000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbdc0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e841dbdd8 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841dbe30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841dbea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841dbe30 blob([0x35b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbea0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e841dbe30) get_ref 0x0~2000 Blob(0x558e841dbe30 blob([0x35b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbea0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e841dbe30) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e841dbe30 blob([0x35b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbea0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e841dbeb8 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841dbf10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841dbf80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841dbf10 blob([0x35d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbf80 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e841dbf10) get_ref 0x0~2000 Blob(0x558e841dbf10 blob([0x35d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbf80 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e841dbf10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e841dbf10 blob([0x35d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbf80 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e841dbf98 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abc000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abc070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abc000 blob([0x35f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc070 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc000) get_ref 0x0~2000 Blob(0x558e83abc000 blob([0x35f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc070 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc000) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83abc000 blob([0x35f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abc070 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83abc088 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abc0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abc150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abc0e0 blob([0x361000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc150 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc0e0) get_ref 0x0~2000 Blob(0x558e83abc0e0 blob([0x361000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc150 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83abc0e0 blob([0x361000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abc150 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83abc168 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abc1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abc230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abc1c0 blob([0x363000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc230 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc1c0) get_ref 0x0~2000 Blob(0x558e83abc1c0 blob([0x363000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc230 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83abc1c0 blob([0x363000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abc230 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83abc248 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abc2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abc310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abc2a0 blob([0x365000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc310 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc2a0) get_ref 0x0~2000 Blob(0x558e83abc2a0 blob([0x365000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc310 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83abc2a0 blob([0x365000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abc310 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83abc328 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abc380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abc3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abc380 blob([0x367000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc3f0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc380) get_ref 0x0~2000 Blob(0x558e83abc380 blob([0x367000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc3f0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc380) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83abc380 blob([0x367000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abc3f0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83abc408 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abc460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abc4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abc460 blob([0x369000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc4d0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc460) get_ref 0x0~2000 Blob(0x558e83abc460 blob([0x369000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc4d0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc460) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e83abc460 blob([0x369000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abc4d0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83abc4e8 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abc540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abc5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abc540 blob([0x36b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc5b0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc540) get_ref 0x0~2000 Blob(0x558e83abc540 blob([0x36b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc5b0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc540) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e83abc540 blob([0x36b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abc5b0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83abc5c8 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abc620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abc690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abc620 blob([0x36d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc690 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc620) get_ref 0x0~2000 Blob(0x558e83abc620 blob([0x36d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc690 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc620) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e83abc620 blob([0x36d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abc690 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83abc6a8 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abc700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abc770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abc700 blob([0x36f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc770 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc700) get_ref 0x0~2000 Blob(0x558e83abc700 blob([0x36f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc770 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc700) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e83abc700 blob([0x36f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abc770 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83abc788 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abc7e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abc850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abc7e0 blob([0x371000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc850 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc7e0) get_ref 0x0~2000 Blob(0x558e83abc7e0 blob([0x371000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc850 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e83abc7e0 blob([0x371000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abc850 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83abc868 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abc8c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abc930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abc8c0 blob([0x373000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc930 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc8c0) get_ref 0x0~2000 Blob(0x558e83abc8c0 blob([0x373000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abc930 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e83abc8c0 blob([0x373000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abc930 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83abc948 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abc9a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abca10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abc9a0 blob([0x375000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abca10 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc9a0) get_ref 0x0~2000 Blob(0x558e83abc9a0 blob([0x375000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abca10 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abc9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e83abc9a0 blob([0x375000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abca10 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83abca28 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abca80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abcaf0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abca80 blob([0x377000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abcaf0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abca80) get_ref 0x0~1000 Blob(0x558e83abca80 blob([0x377000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abcaf0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.blob(0x558e83abca80) get_ref init 0x1000, 1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~1000: 0x0~1000 Blob(0x558e83abca80 blob([0x377000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83abcaf0 sbid 0x0)) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.BufferSpace(0x558e83abcb08 in 0x558e80219180) _discard 0x0~1000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x2b000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.extentmap(0x558e83cb2d88) dirty_range mark inline shard dirty 2019-12-07T19:34:04.664+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# 0x0~2b000 = 0 2019-12-07T19:34:04.664+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.664+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.5s4_head 4#1:a7686634:::smithi13140796-35:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.664+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.0000000027.00000000000000000008' <- 0000000027.00000000000000000008 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.664+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.664+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56c00 cost 270804 (23 ios * 4000 + 178804 bytes) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56c00 onodes 0x558e83cb2c80 shared_blobs 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.extentmap(0x558e83cb2d88) update 4#1:a7686634:::smithi13140796-35:head# 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore.extentmap(0x558e83cb2d88) update inline shard 418 bytes from 22 extents 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 4#1:a7686634:::smithi13140796-35:head# is 909 (485 bytes onode + 2 bytes spanning blobs + 422 bytes inline extents) 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56c00 allocated 0x[34d000~2b000] released 0x[] 2019-12-07T19:34:04.664+0000 7fe2c9269700 10 freelist allocate 0x34d000~2b000 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:04.664+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 prepare 2019-12-07T19:34:04.664+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83c56c00 2019-12-07T19:34:04.664+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=23'7 crt=23'7 active m=2 mbc={} ps=[2~1]] op_applied version 27'8 2019-12-07T19:34:04.664+0000 7fe2c9269700 10 osd.0 27 dequeue_op 0x558e84177e40 finish 2019-12-07T19:34:04.664+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.664+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 aio_wait 2019-12-07T19:34:04.664+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56c00 2019-12-07T19:34:04.664+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 io_done 2019-12-07T19:34:04.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2603 < min_flush_size 524288 2019-12-07T19:34:04.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2c80 had 1 2019-12-07T19:34:04.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:04.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2626 < min_flush_size 524288 2019-12-07T19:34:04.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2626 < min_flush_size 524288 2019-12-07T19:34:04.664+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc6e3d~a42 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.664+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc6e3d 0xc6e3d~a42 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.664+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.664+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x96e3d 2019-12-07T19:34:04.664+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xe3d 2019-12-07T19:34:04.664+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.664+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x87f and padding block with 0x781 2019-12-07T19:34:04.664+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc787f 2019-12-07T19:34:04.664+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.664+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.664+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.664+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000244533s (0.000116239s flush + 0.000128294s kv commit) 2019-12-07T19:34:04.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56c00> 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 kv_submitted 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56c00 2019-12-07T19:34:04.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 finishing 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56c00 onodes 0x558e83cb2c80 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abc088 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a1b0 space 0x558e83abc088 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abc168 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a240 space 0x558e83abc168 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abc248 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a2d0 space 0x558e83abc248 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abc328 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a360 space 0x558e83abc328 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abc408 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a3f0 space 0x558e83abc408 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abc4e8 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a480 space 0x558e83abc4e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abc5c8 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a510 space 0x558e83abc5c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abc6a8 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a5a0 space 0x558e83abc6a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abc788 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a630 space 0x558e83abc788 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abc868 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a6c0 space 0x558e83abc868 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abc948 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a750 space 0x558e83abc948 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abca28 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a7e0 space 0x558e83abca28 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abcb08 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a870 space 0x558e83abcb08 0x0~1000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6f358 in 0x558e80219180) _finish_write discard buffer(0x558e83d11680 space 0x558e83e6f358 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6f4a8 in 0x558e80219180) _finish_write discard buffer(0x558e83d10630 space 0x558e83e6f4a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6f588 in 0x558e80219180) _finish_write discard buffer(0x558e83d10e10 space 0x558e83e6f588 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6f748 in 0x558e80219180) _finish_write discard buffer(0x558e83d10a20 space 0x558e83e6f748 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841db7b8 in 0x558e80219180) _finish_write discard buffer(0x558e83c8fb00 space 0x558e841db7b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841dbcf8 in 0x558e80219180) _finish_write discard buffer(0x558e83d11710 space 0x558e841dbcf8 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841dbdd8 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a000 space 0x558e841dbdd8 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841dbeb8 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a090 space 0x558e841dbeb8 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841dbf98 in 0x558e80219180) _finish_write discard buffer(0x558e83f0a120 space 0x558e841dbf98 0x0~2000 writing nocache) 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56c00 done 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:04.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56c00 [] 2019-12-07T19:34:04.664+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.664+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.665+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 27/26 ECSubWriteReply(tid=34, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e83ca8600 con 0x558e818d6000 2019-12-07T19:34:04.665+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.667+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 85 ==== MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=61, reqid=client.4540.0:207, at_version=0'0, trim_to=0'0, roll_forward_to=27'10)) v2 ==== 1028+0+0 (unknown 0 0 0) 0x558e819d9800 con 0x558e818d7000 2019-12-07T19:34:04.667+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e81942b00 prio 127 cost 0 latency 0.000027 epoch 27 MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=61, reqid=client.4540.0:207, at_version=0'0, trim_to=0'0, roll_forward_to=27'10)) v2 2019-12-07T19:34:04.667+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=61, reqid=client.4540.0:207, at_version=0'0, trim_to=0'0, roll_forward_to=27'10)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=61, reqid=client.4540.0:207, at_version=0'0, trim_to=0'0, roll_forward_to=27'10)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=61, reqid=client.4540.0:207, at_version=0'0, trim_to=0'0, roll_forward_to=27'10)) v2) prio 127 cost 0 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 osd.0 27 dequeue_op 0x558e81942b00 prio 127 cost 0 latency 0.000075 MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=61, reqid=client.4540.0:207, at_version=0'0, trim_to=0'0, roll_forward_to=27'10)) v2 pg pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.fs1 27/26 ECSubWrite(tid=61, reqid=client.4540.0:207, at_version=0'0, trim_to=0'0, roll_forward_to=27'10)) v2 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'6 active+remapped mbc={} ps=[2~1]] append_log log((0'0,27'10], crt=21'6) [] 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'7 (21'2) clone 1:f7919c75:::smithi13140796-20:4 by unknown.0.0:0 2019-12-07T19:33:58.665983+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'8 (21'2) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:205 2019-12-07T19:34:01.660134+0000 0 ObjectCleanRegions clean_offsets: [0~327680,966656~18446744073708584959], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'9 (27'8) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:206 2019-12-07T19:34:01.662593+0000 0 ObjectCleanRegions clean_offsets: [0~966656,1343488~18446744073708208127], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] rollforward: entry=27'10 (27'9) modify 1:f7919c75:::smithi13140796-20:head by client.4540.0:207 2019-12-07T19:34:01.662612+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] append_log approx pg log length = 10 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.667+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e8195fb00 seq 27 2019-12-07T19:34:04.667+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195fb00 cost 5421 (1 ios * 4000 + 1421 bytes) 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195fb00 onodes shared_blobs 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195fb00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 prepare 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195fb00 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 io_done 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 osd.0 27 dequeue_op 0x558e81942b00 finish 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.667+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.667+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.667+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1232 < min_flush_size 524288 2019-12-07T19:34:04.667+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:04.667+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1255 < min_flush_size 524288 2019-12-07T19:34:04.667+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1255 < min_flush_size 524288 2019-12-07T19:34:04.667+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.667+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc787f~4e7 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.667+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc787f 0xc787f~4e7 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.667+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.667+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9787f 2019-12-07T19:34:04.667+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x87f 2019-12-07T19:34:04.667+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.667+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd66 and padding block with 0x29a 2019-12-07T19:34:04.667+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc7d66 2019-12-07T19:34:04.667+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.667+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.667+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.667+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.667+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000234257s (0.000000170s flush + 0.000234087s kv commit) 2019-12-07T19:34:04.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.667+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195fb00> 2019-12-07T19:34:04.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 kv_submitted 2019-12-07T19:34:04.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195fb00 2019-12-07T19:34:04.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 finishing 2019-12-07T19:34:04.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195fb00 onodes 2019-12-07T19:34:04.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 done 2019-12-07T19:34:04.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:04.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195fb00 [] 2019-12-07T19:34:04.667+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.667+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.667+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 86 ==== MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=62, to_read={1:f7919c75:::smithi13140796-20:head=0,4096,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8417cc80 con 0x558e818d7000 2019-12-07T19:34:04.667+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 27/26 ECSubWriteReply(tid=61, last_complete=27'10, committed=1, applied=1)) v2 -- 0x558e81964c00 con 0x558e818d7000 2019-12-07T19:34:04.667+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e819431e0 prio 127 cost 0 latency 0.000018 epoch 27 MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=62, to_read={1:f7919c75:::smithi13140796-20:head=0,4096,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.667+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=62, to_read={1:f7919c75:::smithi13140796-20:head=0,4096,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=62, to_read={1:f7919c75:::smithi13140796-20:head=0,4096,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=62, to_read={1:f7919c75:::smithi13140796-20:head=0,4096,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 osd.0 27 dequeue_op 0x558e819431e0 prio 127 cost 0 latency 0.000051 MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=62, to_read={1:f7919c75:::smithi13140796-20:head=0,4096,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=62, to_read={1:f7919c75:::smithi13140796-20:head=0,4096,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.667+0000 7fe2c8267700 25 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.667+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x0~1000 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x52000 (335872) 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac49a0 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x0~1000 = 4096 2019-12-07T19:34:04.667+0000 7fe2c8267700 20 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:f7919c75:::smithi13140796-20:head 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:f7919c75:::smithi13140796-20:head 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:34:04.667+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 335872 2019-12-07T19:34:04.668+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key 2019-12-07T19:34:04.668+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key = 0 2019-12-07T19:34:04.668+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.fs0 27/26 ECSubReadReply(tid=62, attrs_read=0)) v2 -- 0x558e8417cf00 con 0x558e818d7000 2019-12-07T19:34:04.668+0000 7fe2c8267700 10 osd.0 27 dequeue_op 0x558e819431e0 finish 2019-12-07T19:34:04.668+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.668+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.668+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 146 ==== MOSDECSubOpRead(1.es4 27/26 ECSubRead(tid=112, to_read={1:76d0c63f:::smithi13140796-38:head=0,262144,0}, subchunks={1:76d0c63f:::smithi13140796-38:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (unknown 0 0 0) 0x558e8417cf00 con 0x558e818d7c00 2019-12-07T19:34:04.668+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83b0a840 prio 3 cost 0 latency 0.000022 epoch 27 MOSDECSubOpRead(1.es4 27/26 ECSubRead(tid=112, to_read={1:76d0c63f:::smithi13140796-38:head=0,262144,0}, subchunks={1:76d0c63f:::smithi13140796-38:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.668+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpRead(1.es4 27/26 ECSubRead(tid=112, to_read={1:76d0c63f:::smithi13140796-38:head=0,262144,0}, subchunks={1:76d0c63f:::smithi13140796-38:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpRead(1.es4 27/26 ECSubRead(tid=112, to_read={1:76d0c63f:::smithi13140796-38:head=0,262144,0}, subchunks={1:76d0c63f:::smithi13140796-38:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGOpItem(op=MOSDECSubOpRead(1.es4 27/26 ECSubRead(tid=112, to_read={1:76d0c63f:::smithi13140796-38:head=0,262144,0}, subchunks={1:76d0c63f:::smithi13140796-38:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e81a98000 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 osd.0 27 dequeue_op 0x558e83b0a840 prio 3 cost 0 latency 0.000506 MOSDECSubOpRead(1.es4 27/26 ECSubRead(tid=112, to_read={1:76d0c63f:::smithi13140796-38:head=0,262144,0}, subchunks={1:76d0c63f:::smithi13140796-38:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.es4 27/26 ECSubRead(tid=112, to_read={1:76d0c63f:::smithi13140796-38:head=0,262144,0}, subchunks={1:76d0c63f:::smithi13140796-38:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.669+0000 7fe2c4a60700 25 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x200~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x200~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28b60 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28b60 sbid 0x0)) need {<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28b60 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83b28b78 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e811714d0 space 0x558e83b28b78 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x200~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x600~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x600~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28b60 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x600~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xa00~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa00~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28b60 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xa00~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xe00~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe00~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28b60 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xe00~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1200~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1200~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28b60 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28b60 sbid 0x0)) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28b60 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83b28b78 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0aab0 space 0x558e83b28b78 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1200~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1600~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1600~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28b60 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1600~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1a00~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28b60 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1a00~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1e00~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e88c0 blob([0x19d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28b60 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1e00~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2200~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2200~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c649a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c649a0 sbid 0x0)) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c649a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83c649b8 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0aa20 space 0x558e83c649b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2200~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2600~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2600~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c649a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2600~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2a00~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c649a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2a00~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2e00~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c649a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2e00~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3200~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3200~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c649a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c649a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c649a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83c649b8 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0a990 space 0x558e83c649b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3200~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3600~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3600~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c649a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3600~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3a00~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c649a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3a00~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3e00~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64a10 blob([0x19f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c649a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3e00~200 = 512 2019-12-07T19:34:04.669+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x4200~200 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4200~200 size 0x26000 (155648) 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81079810 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81079810 sbid 0x0)) need {<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.669+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81079810 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e81079828 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0acf0 space 0x558e81079828 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x4200~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x4600~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4600~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81079810 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x4600~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x4a00~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81079810 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x4a00~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x4e00~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81079810 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x4e00~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x5200~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5200~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81079810 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81079810 sbid 0x0)) need {<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x5200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81079810 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e81079828 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0ad80 space 0x558e81079828 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x5200~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x5600~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5600~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81079810 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x5600~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x5a00~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81079810 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x5a00~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x5e00~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c648c0 blob([0x1a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81079810 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x5e00~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x6200~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6200~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fe30 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fe30 sbid 0x0)) need {<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fe30 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e8194fe48 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0ae10 space 0x558e8194fe48 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x6200~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x6600~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6600~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fe30 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x6600~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x6a00~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fe30 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x6a00~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x6e00~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fe30 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x6e00~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x7200~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7200~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fe30 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fe30 sbid 0x0)) need {<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x7200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fe30 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e8194fe48 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0aea0 space 0x558e8194fe48 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x7200~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x7600~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7600~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fe30 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x7600~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x7a00~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fe30 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x7a00~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x7e00~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194fb90 blob([0x1a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fe30 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x7e00~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x8200~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8200~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da89a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da89a0 sbid 0x0)) need {<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da89a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da89b8 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0af30 space 0x558e83da89b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x8200~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x8600~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8600~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da89a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x8600~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x8a00~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da89a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x8a00~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x8e00~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da89a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x8e00~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x9200~200 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9200~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da89a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da89a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x9200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.670+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 31 ==== MOSDECSubOpWrite(1.5s4 27/26 ECSubWrite(tid=35, reqid=client.4540.0:203, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2 ==== 1080+0+0 (unknown 0 0 0) 0x558e83fed800 con 0x558e818d6000 2019-12-07T19:34:04.670+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e83b0bce0 prio 127 cost 0 latency 0.000024 epoch 27 MOSDECSubOpWrite(1.5s4 27/26 ECSubWrite(tid=35, reqid=client.4540.0:203, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2 2019-12-07T19:34:04.670+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGOpItem(op=MOSDECSubOpWrite(1.5s4 27/26 ECSubWrite(tid=35, reqid=client.4540.0:203, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da89a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.670+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da89b8 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.670+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDECSubOpWrite(1.5s4 27/26 ECSubWrite(tid=35, reqid=client.4540.0:203, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0afc0 space 0x558e83da89b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.670+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.670+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x9200~200 = 512 2019-12-07T19:34:04.670+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x9600~200 2019-12-07T19:34:04.670+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDECSubOpWrite(1.5s4 27/26 ECSubWrite(tid=35, reqid=client.4540.0:203, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2) prio 127 cost 0 e27) pg 0x558e83d07000 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9600~200 size 0x26000 (155648) 2019-12-07T19:34:04.670+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da89a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x9600~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x9a00~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.670+0000 7fe2c5261700 10 osd.0 27 dequeue_op 0x558e83b0bce0 prio 127 cost 0 latency 0.000075 MOSDECSubOpWrite(1.5s4 27/26 ECSubWrite(tid=35, reqid=client.4540.0:203, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2 pg pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 active m=2 mbc={} ps=[2~1]] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da89a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x9a00~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x9e00~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8930 blob([0x1a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da89a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.671+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 active m=2 mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.5s4 27/26 ECSubWrite(tid=35, reqid=client.4540.0:203, at_version=0'0, trim_to=0'0, roll_forward_to=27'8)) v2 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x9e00~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xa200~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa200~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8a80 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8a80 sbid 0x0)) need {<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.671+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'7 active m=2 mbc={} ps=[2~1]] append_log log((0'0,27'8], crt=23'7) [] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.671+0000 7fe2c5261700 20 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active m=2 mbc={} ps=[2~1]] rollforward: entry=27'8 (0'0) modify 1:a7686634:::smithi13140796-35:head by client.4540.0:203 0.000000 0 ObjectCleanRegions clean_offsets: [704512~18446744073708847103], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.671+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active m=2 mbc={} ps=[2~1]] append_log approx pg log length = 8 2019-12-07T19:34:04.671+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active m=2 mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.671+0000 7fe2c5261700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.671+0000 7fe2c5261700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.671+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:04.671+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e83c57800 seq 7 2019-12-07T19:34:04.671+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.671+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:04.671+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.671+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.671+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.671+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c57800 cost 5473 (1 ios * 4000 + 1473 bytes) 2019-12-07T19:34:04.671+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c57800 onodes shared_blobs 2019-12-07T19:34:04.671+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c57800 allocated 0x[] released 0x[] 2019-12-07T19:34:04.671+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 prepare 2019-12-07T19:34:04.671+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c57800 2019-12-07T19:34:04.671+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 io_done 2019-12-07T19:34:04.671+0000 7fe2c5261700 10 osd.0 27 dequeue_op 0x558e83b0bce0 finish 2019-12-07T19:34:04.671+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.671+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.671+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.671+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 666 < min_flush_size 524288 2019-12-07T19:34:04.671+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1295 < min_flush_size 524288 2019-12-07T19:34:04.671+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:04.671+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1318 < min_flush_size 524288 2019-12-07T19:34:04.671+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1318 < min_flush_size 524288 2019-12-07T19:34:04.671+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.671+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc7d66~526 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.671+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc7d66 0xc7d66~526 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.671+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.671+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x97d66 2019-12-07T19:34:04.671+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd66 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8a80 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.671+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.671+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x28c and padding block with 0xd74 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da8a98 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b050 space 0x558e83da8a98 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xa200~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xa600~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa600~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8a80 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xa600~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xaa00~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xaa00~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8a80 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xaa00~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xae00~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xae00~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8a80 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xae00~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xb200~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb200~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8a80 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8a80 sbid 0x0)) need {<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xb200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.671+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 147 ==== MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=113, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8402e500 con 0x558e818d7c00 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.671+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83b0be40 prio 3 cost 0 latency 0.000018 epoch 27 MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=113, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.671+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=113, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=113, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=113, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e8122e000 2019-12-07T19:34:04.671+0000 7fe2c8267700 10 osd.0 27 dequeue_op 0x558e83b0be40 prio 3 cost 0 latency 0.000071 MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=113, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.671+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=113, to_read={1:ec1285ab:::smithi13140796-19:head=0,262144,0}, subchunks={1:ec1285ab:::smithi13140796-19:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.671+0000 7fe2c8267700 25 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.671+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 0x0~40000 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~40000 size 0x13000 (77824) 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8e70 blob([0x8b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8150 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af80e0 blob([0x8d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8070 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5ab0 blob([0x8f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8f50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e90a0 blob([0x91000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af81c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8230 blob([0x93000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af82a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8310 blob([0x95000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8380 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8a80 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af83f0 blob([0x97000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8460 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da8a98 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b0e0 space 0x558e83da8a98 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af84d0 blob([0x99000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8540 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af85b0 blob([0x9b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8620 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xb200~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xb600~200 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8690 blob([0x9d000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83af8700 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb600~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 0x0~40000 = 77824 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8a80 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xb600~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xba00~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xba00~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=77824 len=77824 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8a80 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xba00~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xbe00~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xbe00~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8a10 blob([0x1a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8a80 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xbe00~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xc200~200 2019-12-07T19:34:04.671+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc200~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8b60 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8b60 sbid 0x0)) need {<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.671+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 77824 2019-12-07T19:34:04.671+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# hinfo_key 2019-12-07T19:34:04.671+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ec1285ab:::smithi13140796-19:head# hinfo_key = 0 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:ec1285ab:::smithi13140796-19:head 2019-12-07T19:34:04.671+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 27/26 ECSubReadReply(tid=113, attrs_read=0)) v2 -- 0x558e8417cc80 con 0x558e818d7c00 2019-12-07T19:34:04.671+0000 7fe2c8267700 10 osd.0 27 dequeue_op 0x558e83b0be40 finish 2019-12-07T19:34:04.671+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.671+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc828c 2019-12-07T19:34:04.671+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.671+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.671+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.671+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.671+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000405928s (0.000000102s flush + 0.000405826s kv commit) 2019-12-07T19:34:04.671+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c57800> 2019-12-07T19:34:04.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.671+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 kv_submitted 2019-12-07T19:34:04.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c57800 2019-12-07T19:34:04.671+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 finishing 2019-12-07T19:34:04.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c57800 onodes 2019-12-07T19:34:04.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57800 done 2019-12-07T19:34:04.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:04.671+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c57800 [] 2019-12-07T19:34:04.671+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.671+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDECSubOpWriteReply(1.5s0 27/26 ECSubWriteReply(tid=35, last_complete=21'1, committed=1, applied=1)) v2 -- 0x558e81955600 con 0x558e818d6000 2019-12-07T19:34:04.671+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.671+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8b60 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da8b78 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b170 space 0x558e83da8b78 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xc200~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xc600~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc600~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8b60 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xc600~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xca00~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xca00~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8b60 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xca00~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xce00~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xce00~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8b60 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xce00~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xd200~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd200~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8b60 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8b60 sbid 0x0)) need {<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xd200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8b60 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da8b78 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b200 space 0x558e83da8b78 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xd200~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xd600~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd600~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8b60 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xd600~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xda00~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xda00~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8b60 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xda00~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xde00~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xde00~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8af0 blob([0x1a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8b60 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xde00~200 = 512 2019-12-07T19:34:04.671+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xe200~200 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe200~200 size 0x26000 (155648) 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8c40 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8c40 sbid 0x0)) need {<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8c40 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da8c58 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.671+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b290 space 0x558e83da8c58 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xe200~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xe600~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe600~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8c40 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xe600~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xea00~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xea00~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8c40 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xea00~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xee00~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xee00~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8c40 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xee00~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xf200~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf200~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8c40 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8c40 sbid 0x0)) need {<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xf200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8c40 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da8c58 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b320 space 0x558e83da8c58 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xf200~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xf600~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf600~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8c40 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xf600~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xfa00~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfa00~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8c40 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xfa00~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xfe00~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfe00~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8bd0 blob([0x1ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8c40 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0xfe00~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x10200~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10200~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ff10 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ff10 sbid 0x0)) need {<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ff10 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83d0ff28 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b3b0 space 0x558e83d0ff28 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x10200~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x10600~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10600~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ff10 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x10600~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x10a00~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ff10 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x10a00~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x10e00~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ff10 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x10e00~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x11200~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11200~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ff10 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ff10 sbid 0x0)) need {<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x11200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ff10 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83d0ff28 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b440 space 0x558e83d0ff28 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x11200~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x11600~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11600~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ff10 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x11600~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x11a00~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ff10 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x11a00~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x11e00~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b29030 blob([0x1ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ff10 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x11e00~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x12200~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12200~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8d20 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8d20 sbid 0x0)) need {<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x12200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8d20 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da8d38 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b4d0 space 0x558e83da8d38 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x12200~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x12600~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12600~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8d20 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x12600~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x12a00~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8d20 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x12a00~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x12e00~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8d20 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x12e00~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x13200~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13200~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8d20 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8d20 sbid 0x0)) need {<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x13200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8d20 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da8d38 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b560 space 0x558e83da8d38 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x13200~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x13600~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13600~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8d20 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x13600~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x13a00~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8d20 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x13a00~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x13e00~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8cb0 blob([0x1af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8d20 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x13e00~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x14200~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14200~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8e00 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8e00 sbid 0x0)) need {<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x14200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8e00 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da8e18 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b5f0 space 0x558e83da8e18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x14200~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x14600~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14600~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8e00 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.672+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x14600~200 = 512 2019-12-07T19:34:04.672+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x14a00~200 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.672+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8e00 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x14a00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x14e00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8e00 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x14e00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x15200~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15200~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8e00 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8e00 sbid 0x0)) need {<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x15200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8e00 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da8e18 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b680 space 0x558e83da8e18 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x15200~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x15600~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15600~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8e00 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x15600~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x15a00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8e00 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x15a00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x15e00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8d90 blob([0x1b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8e00 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x15e00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x16200~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16200~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8ee0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8ee0 sbid 0x0)) need {<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x16200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8ee0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da8ef8 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b710 space 0x558e83da8ef8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x16200~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x16600~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16600~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8ee0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x16600~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x16a00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8ee0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x16a00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x16e00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8ee0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x16e00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x17200~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17200~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8ee0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8ee0 sbid 0x0)) need {<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x17200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8ee0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da8ef8 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b7a0 space 0x558e83da8ef8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x17200~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x17600~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17600~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8ee0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x17600~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x17a00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8ee0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x17a00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x17e00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8e70 blob([0x1b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8ee0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x17e00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x18200~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18200~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8fc0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8fc0 sbid 0x0)) need {<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x18200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8fc0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da8fd8 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b830 space 0x558e83da8fd8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x18200~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x18600~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18600~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8fc0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x18600~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x18a00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8fc0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x18a00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x18e00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8fc0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x18e00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x19200~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19200~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8fc0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8fc0 sbid 0x0)) need {<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x19200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8fc0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da8fd8 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b8c0 space 0x558e83da8fd8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x19200~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x19600~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19600~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8fc0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x19600~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x19a00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8fc0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x19a00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x19e00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da8f50 blob([0x1b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da8fc0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x19e00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1a200~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a200~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da90a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da90a0 sbid 0x0)) need {<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1a200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da90a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da90b8 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b950 space 0x558e83da90b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1a200~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1a600~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a600~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da90a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1a600~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1aa00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1aa00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da90a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1aa00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1ae00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ae00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da90a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1ae00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1b200~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b200~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da90a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da90a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1b200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da90a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da90b8 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0b9e0 space 0x558e83da90b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1b200~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1b600~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b600~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da90a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1b600~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1ba00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ba00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da90a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1ba00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1be00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1be00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9030 blob([0x1b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da90a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1be00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1c200~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c200~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9180 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9180 sbid 0x0)) need {<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1c200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9180 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da9198 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0ba70 space 0x558e83da9198 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1c200~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1c600~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c600~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9180 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1c600~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1ca00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ca00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9180 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1ca00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1ce00~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ce00~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9180 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1ce00~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1d200~200 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d200~200 size 0x26000 (155648) 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9180 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9180 sbid 0x0)) need {<0x1000, 0x1000> : [0x1d200:1200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1d200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9180 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1d200:1200~200]} 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da9198 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.673+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0bb00 space 0x558e83da9198 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.673+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1d200~200 = 512 2019-12-07T19:34:04.673+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1d600~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d600~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9180 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1d600~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1da00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1da00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9180 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1da00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1de00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1de00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9110 blob([0x1b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9180 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1de00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1e200~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e200~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9260 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9260 sbid 0x0)) need {<0x0, 0x1000> : [0x1e200:200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1e200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9260 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1e200:200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da9278 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0bb90 space 0x558e83da9278 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1e200~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1e600~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e600~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9260 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1e600~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1ea00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ea00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9260 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1ea00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1ee00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ee00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9260 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1ee00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1f200~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f200~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9260 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9260 sbid 0x0)) need {<0x1000, 0x1000> : [0x1f200:1200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1f200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9260 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1f200:1200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da9278 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0bc20 space 0x558e83da9278 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1f200~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1f600~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f600~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9260 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1f600~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1fa00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fa00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9260 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1fa00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1fe00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fe00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da91f0 blob([0x1bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9260 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x1fe00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x20200~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20200~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9340 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9340 sbid 0x0)) need {<0x0, 0x1000> : [0x20200:200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x20200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9340 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x20200:200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da9358 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0bcb0 space 0x558e83da9358 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x20200~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x20600~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20600~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9340 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x20600~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x20a00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9340 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x20a00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x20e00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9340 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x20e00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x21200~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21200~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9340 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9340 sbid 0x0)) need {<0x1000, 0x1000> : [0x21200:1200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x21200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9340 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x21200:1200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da9358 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0bd40 space 0x558e83da9358 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x21200~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x21600~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21600~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9340 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x21600~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x21a00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9340 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x21a00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x21e00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da92d0 blob([0x1bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9340 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x21e00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x22200~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22200~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9420 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9420 sbid 0x0)) need {<0x0, 0x1000> : [0x22200:200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x22200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9420 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x22200:200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da9438 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0bdd0 space 0x558e83da9438 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x22200~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x22600~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22600~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9420 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x22600~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x22a00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9420 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x22a00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x22e00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9420 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x22e00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x23200~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23200~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9420 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9420 sbid 0x0)) need {<0x1000, 0x1000> : [0x23200:1200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x23200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9420 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x23200:1200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da9438 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0be60 space 0x558e83da9438 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x23200~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x23600~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23600~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9420 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x23600~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x23a00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9420 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x23a00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x23e00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da93b0 blob([0x1bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9420 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x23e00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x24200~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24200~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9500 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9500 sbid 0x0)) need {<0x0, 0x1000> : [0x24200:200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x24200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9500 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x24200:200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da9518 in 0x558e80219ea0) _discard 0x0~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e83f0bef0 space 0x558e83da9518 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x24200~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x24600~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24600~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9500 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x24600~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x24a00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9500 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x24a00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x24e00~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9500 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x24e00~200 = 512 2019-12-07T19:34:04.674+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x25200~200 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25200~200 size 0x26000 (155648) 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9500 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9500 sbid 0x0)) need {<0x1000, 0x1000> : [0x25200:1200~200]} 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x25200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.674+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9500 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x25200:1200~200]} 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore.BufferSpace(0x558e83da9518 in 0x558e80219ea0) _discard 0x1000~1000 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 _add level 1 near 0 on buffer(0x558e81171b90 space 0x558e83da9518 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x25200~200 = 512 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x25600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9500 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x25600~200 = 512 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x25a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9500 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x25a00~200 = 512 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x25e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83da9490 blob([0x1c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9500 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x25e00~200 = 512 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x26200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x26200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x26600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x26600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x26a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x26a00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x26e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x26e00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x27200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x27200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x27600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x27600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x27a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x27a00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x27e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x27e00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x28200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x28200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x28600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x28600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x28a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x28a00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x28e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x28e00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x29200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x29200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x29600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x29600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x29a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x29a00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x29e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x29e00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2a200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2a200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2a600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2a600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2aa00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2aa00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2aa00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2ae00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ae00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2ae00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2b200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2b200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2b600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2b600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2ba00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ba00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2ba00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2be00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2be00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2be00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2c200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2c200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2c600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2c600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2ca00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ca00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2ca00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2ce00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ce00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2ce00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2d200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2d200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2d600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2d600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2da00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2da00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2da00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2de00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2de00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2de00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2e200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2e200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2e600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2e600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2ea00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ea00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2ea00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2ee00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ee00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2ee00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2f200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2f200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2f600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2f600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2fa00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fa00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2fa00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2fe00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fe00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x2fe00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x30200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x30200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x30600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x30600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x30a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x30a00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x30e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x30e00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x31200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x31200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x31600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x31600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x31a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x31a00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x31e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x31e00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x32200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x32200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x32600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x32600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x32a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x32a00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x32e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x32e00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x33200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x33200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x33600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x33600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x33a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x33a00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x33e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x33e00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x34200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x34200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x34600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x34600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x34a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x34a00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x34e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x34e00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x35200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x35200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x35600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x35600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x35a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x35a00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x35e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x35e00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x36200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x36200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x36600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x36600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x36a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x36a00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x36e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x36e00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x37200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x37200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x37600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x37600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x37a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x37a00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x37e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x37e00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x38200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x38200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x38600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x38600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x38a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x38a00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x38e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x38e00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x39200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x39200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x39600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x39600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x39a00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39a00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x39a00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x39e00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39e00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x39e00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3a200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3a200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3a600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3a600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3aa00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3aa00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3aa00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3ae00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ae00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3ae00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3b200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3b200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3b600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3b600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3ba00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ba00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3ba00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3be00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3be00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3be00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3c200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3c200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3c600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3c600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3ca00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ca00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3ca00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3ce00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ce00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3ce00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3d200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3d200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3d600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3d600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3da00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3da00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3da00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3de00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3de00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3de00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3e200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3e200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3e600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3e600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3ea00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ea00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3ea00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3ee00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ee00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3ee00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3f200~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f200~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3f200~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3f600~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f600~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3f600~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3fa00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fa00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3fa00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3fe00~200 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fe00~200 size 0x26000 (155648) 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 0x3fe00~200 = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=77824 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:76d0c63f:::smithi13140796-38:head 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:76d0c63f:::smithi13140796-38:head 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 155648 2019-12-07T19:34:04.676+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# hinfo_key 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.es4_head 4#1:76d0c63f:::smithi13140796-38:head# hinfo_key = 0 2019-12-07T19:34:04.676+0000 7fe2c4a60700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.es0 27/26 ECSubReadReply(tid=112, attrs_read=0)) v2 -- 0x558e811dc000 con 0x558e818d7c00 2019-12-07T19:34:04.676+0000 7fe2c4a60700 10 osd.0 27 dequeue_op 0x558e83b0a840 finish 2019-12-07T19:34:04.676+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.681+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 148 ==== MRecoveryReserve(1.es4 RELEASE e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e819b0780 con 0x558e818d7c00 2019-12-07T19:34:04.681+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.es4 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.681+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.681+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.681+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.681+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.681+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558e81a98000 2019-12-07T19:34:04.681+0000 7fe2c8a68700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.681+0000 7fe2c8a68700 5 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.682076+0000 1 0.000033 2019-12-07T19:34:04.681+0000 7fe2cea74700 15 osd.0 27 enqueue_peering_evt 1.3s5 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.681+0000 7fe2cea74700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.681+0000 7fe2c8a68700 5 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.681+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:04.681+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:04.681+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.681+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.681+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.681+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e81231000 2019-12-07T19:34:04.681+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.681+0000 7fe2c6263700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MRecoveryReserve(1.3s0 GRANT e27/27) v3 -- 0x558e841983c0 con 0x558e818fe400 2019-12-07T19:34:04.681+0000 7fe2c6263700 5 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.682185+0000 1 0.000037 2019-12-07T19:34:04.681+0000 7fe2c6263700 5 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.681+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:04.681+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.681+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 87 ==== MRecoveryReserve(1.2s0 GRANT e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e819b0780 con 0x558e818d7000 2019-12-07T19:34:04.681+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.681+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.681+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 149 ==== MRecoveryReserve(1.9s3 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e819b0780 con 0x558e818d7c00 2019-12-07T19:34:04.681+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.9s3 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.681+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.681+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.681+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.681+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.681+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558e8192b000 2019-12-07T19:34:04.681+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.681+0000 7fe2c7265700 5 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.682557+0000 4 0.000100 2019-12-07T19:34:04.681+0000 7fe2c7265700 5 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.681+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:04.681+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.681+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 150 ==== MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=114, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e811dc000 con 0x558e818d7c00 2019-12-07T19:34:04.682+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edc840 prio 3 cost 0 latency 0.000026 epoch 27 MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=114, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.682+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=114, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=114, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGOpItem(op=MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=114, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e8122e000 2019-12-07T19:34:04.682+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e83edc840 prio 3 cost 0 latency 0.000075 MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=114, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.682+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.7s3 27/26 ECSubRead(tid=114, to_read={1:ee170468:::smithi13140796-25:head=0,262144,0}, subchunks={1:ee170468:::smithi13140796-25:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.682+0000 7fe2c425f700 25 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.682+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 0x0~40000 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~40000 size 0x19000 (102400) 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f7a0 blob([0xe2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac59d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4a80 blob([0xe4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9a40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9b90 blob([0xe6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9c00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9c70 blob([0xe8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9ce0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.682+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 151 ==== MOSDPGRecoveryDelete(1.8s5 e27,26 1:14fc029d:::smithi13140796-41:head,21'13) v2 ==== 123+0+0 (unknown 0 0 0) 0x558e81964a00 con 0x558e818d7c00 2019-12-07T19:34:04.682+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edcf20 prio 196 cost 1000 latency 0.000012 epoch 27 MOSDPGRecoveryDelete(1.8s5 e27,26 1:14fc029d:::smithi13140796-41:head,21'13) v2 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9d50 blob([0xea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9dc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.682+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGRecoveryDelete(1.8s5 e27,26 1:14fc029d:::smithi13140796-41:head,21'13) v2) prio 196 cost 1000 e27) 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9e30 blob([0xec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9ea0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9f10 blob([0xee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9f80 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e000 blob([0xf0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e070 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e0e0 blob([0xf2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e150 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e1c0 blob([0xf4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e230 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGRecoveryDelete(1.8s5 e27,26 1:14fc029d:::smithi13140796-41:head,21'13) v2) prio 196 cost 1000 e27) queued 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGRecoveryDelete(1.8s5 e27,26 1:14fc029d:::smithi13140796-41:head,21'13) v2) prio 196 cost 1000 e27) pg 0x558e841a4000 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e2a0 blob([0xf6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e310 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e83edcf20 prio 196 cost 1000 latency 0.000038 MOSDPGRecoveryDelete(1.8s5 e27,26 1:14fc029d:::smithi13140796-41:head,21'13) v2 pg pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=7 mbc={} ps=[2~1]] 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e380 blob([0xf8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0e3f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0e460 blob([0xfa000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d0e4d0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=7 mbc={} ps=[2~1]] handle_recovery_delete 0x558e83edcf20 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=7 mbc={} ps=[2~1]] remove_missing_object 1:14fc029d:::smithi13140796-41:head 21'13 2019-12-07T19:34:04.682+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 0x0~40000 = 102400 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=102400 len=102400 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e819a1b00 seq 8 2019-12-07T19:34:04.682+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:ee170468:::smithi13140796-25:head 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:14fc029d:::smithi13140796-41:head# key 0x85800000000000000114fc029d21736d'ithi13140796-41!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.682+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:ee170468:::smithi13140796-25:head 2019-12-07T19:34:04.682+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# 2019-12-07T19:34:04.682+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 102400 2019-12-07T19:34:04.682+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# hinfo_key 2019-12-07T19:34:04.682+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.7s3_head 3#1:ee170468:::smithi13140796-25:head# hinfo_key = 0 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 5#1:14fc029d:::smithi13140796-41:head# 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1b00 cost 4212 (1 ios * 4000 + 212 bytes) 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1b00 onodes shared_blobs 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:ee170468:::smithi13140796-25:head 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1b00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.682+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.7s0 27/26 ECSubReadReply(tid=114, attrs_read=0)) v2 -- 0x558e811dc780 con 0x558e818d7c00 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 prepare 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1b00 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 io_done 2019-12-07T19:34:04.682+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e83edc840 finish 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e83edcf20 finish 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.682+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc828c~2e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc828c 0xc828c~2e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9828c 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x28c 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x2ba and padding block with 0xd46 2019-12-07T19:34:04.682+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc82ba 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000145275s (0.000000168s flush + 0.000145107s kv commit) 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1b00> 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.682+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 kv_submitted 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1b00 2019-12-07T19:34:04.682+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 finishing 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1b00 onodes 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1b00 done 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.682+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1b00 [] 2019-12-07T19:34:04.682+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=7 mbc={} ps=[2~1]] on_local_recover: 1:14fc029d:::smithi13140796-41:head 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=7 mbc={} ps=[2~1]] got missing 1:14fc029d:::smithi13140796-41:head v 21'13 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=6 mbc={} ps=[2~1]] last_complete now 21'3 log.complete_to 21'4 2019-12-07T19:34:04.682+0000 7fe2cba6e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e811fb800 seq 9 2019-12-07T19:34:04.682+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x000000000000045f'.missing/0000000000000001.82F3049B.head.smithi13140796-41..' <- missing/0000000000000001.82F3049B.head.smithi13140796-41.. 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.682+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fb800 cost 5527 (1 ios * 4000 + 1527 bytes) 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fb800 onodes shared_blobs 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fb800 allocated 0x[] released 0x[] 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 prepare 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fb800 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 io_done 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=6 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.682+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1303 < min_flush_size 524288 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1326 < min_flush_size 524288 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1326 < min_flush_size 524288 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc82ba~52e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc82ba 0xc82ba~52e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x982ba 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x2ba 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x7e8 and padding block with 0x818 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc87e8 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.682+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000126905s (0.000000088s flush + 0.000126817s kv commit) 2019-12-07T19:34:04.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fb800> 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.682+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 kv_submitted 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fb800 2019-12-07T19:34:04.682+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 finishing 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fb800 onodes 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb800 done 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.682+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fb800 [] 2019-12-07T19:34:04.682+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.682+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.682+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=6 mbc={} ps=[2~1]] recovery_committed_to version 21'3 now ondisk 2019-12-07T19:34:04.682+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGRecoveryDeleteReply(1.8s0 e27,26 1:14fc029d:::smithi13140796-41:head,21'13) v2 -- 0x558e81964c00 con 0x558e818d7c00 2019-12-07T19:34:04.682+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.682+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.682+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.682+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.682+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.682+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.682+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.682+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_recovery_priority recovery priority is 190 2019-12-07T19:34:04.682+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MRecoveryReserve(1.2s1 REQUEST prio: 190 e27/27) v3 -- 0x558e819b05a0 con 0x558e818fe400 2019-12-07T19:34:04.682+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.682+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.684+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 152 ==== pg_update_log_missing(1.8s5 epoch 27/26 rep_tid 115 entries 27'19 (0'0) error 1:14fc029d:::smithi13140796-41:head by client.4540.0:219 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 21'3) v3 ==== 258+0+0 (unknown 0 0 0) 0x558e83c67200 con 0x558e818d7c00 2019-12-07T19:34:04.684+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edc6e0 prio 127 cost 0 latency 0.000022 epoch 27 pg_update_log_missing(1.8s5 epoch 27/26 rep_tid 115 entries 27'19 (0'0) error 1:14fc029d:::smithi13140796-41:head by client.4540.0:219 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 21'3) v3 2019-12-07T19:34:04.684+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=pg_update_log_missing(1.8s5 epoch 27/26 rep_tid 115 entries 27'19 (0'0) error 1:14fc029d:::smithi13140796-41:head by client.4540.0:219 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 21'3) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=pg_update_log_missing(1.8s5 epoch 27/26 rep_tid 115 entries 27'19 (0'0) error 1:14fc029d:::smithi13140796-41:head by client.4540.0:219 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 21'3) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=pg_update_log_missing(1.8s5 epoch 27/26 rep_tid 115 entries 27'19 (0'0) error 1:14fc029d:::smithi13140796-41:head by client.4540.0:219 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 21'3) v3) prio 127 cost 0 e27) pg 0x558e841a4000 2019-12-07T19:34:04.684+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e83edc6e0 prio 127 cost 0 latency 0.000171 pg_update_log_missing(1.8s5 epoch 27/26 rep_tid 115 entries 27'19 (0'0) error 1:14fc029d:::smithi13140796-41:head by client.4540.0:219 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 21'3) v3 pg pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=6 mbc={} ps=[2~1]] 2019-12-07T19:34:04.684+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=6 mbc={} ps=[2~1]] _handle_message: pg_update_log_missing(1.8s5 epoch 27/26 rep_tid 115 entries 27'19 (0'0) error 1:14fc029d:::smithi13140796-41:head by client.4540.0:219 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 21'3) v3 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'18 lc 21'3 (0'0,27'18] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=6 mbc={} ps=[2~1]] do_update_log_missing op_trim_to = -- op_roll_forward_to = 21'3 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 update missing, append 27'19 (0'0) error 1:14fc029d:::smithi13140796-41:head by client.4540.0:219 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'19 lc 21'3 (0'0,27'19] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'18 crt=27'18 active m=6 mbc={} ps=[2~1]] append_log_entries_update_missing trim_to bool = 0 trim_to = 0'0 2019-12-07T19:34:04.684+0000 7fe2cba6e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'19, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.684+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e83e72000 seq 10 2019-12-07T19:34:04.684+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000027.00000000000000000019' <- 0000000027.00000000000000000019 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._fastinfo' <- _fastinfo 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.684+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.684+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72000 cost 4717 (1 ios * 4000 + 717 bytes) 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72000 onodes shared_blobs 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72000 allocated 0x[] released 0x[] 2019-12-07T19:34:04.684+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 prepare 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72000 2019-12-07T19:34:04.684+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 io_done 2019-12-07T19:34:04.684+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.684+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'19 lc 21'3 (0'0,27'19] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'18 crt=27'18 active m=6 mbc={} ps=[2~1]] op_applied version 27'19 2019-12-07T19:34:04.684+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e83edc6e0 finish 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.684+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.684+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 567 < min_flush_size 524288 2019-12-07T19:34:04.684+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.684+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 590 < min_flush_size 524288 2019-12-07T19:34:04.684+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 590 < min_flush_size 524288 2019-12-07T19:34:04.684+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.684+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc87e8~24e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.684+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc87e8 0xc87e8~24e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.684+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.684+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x987e8 2019-12-07T19:34:04.684+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x7e8 2019-12-07T19:34:04.684+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.684+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xa36 and padding block with 0x5ca 2019-12-07T19:34:04.684+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc8a36 2019-12-07T19:34:04.684+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.684+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.684+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.684+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.684+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000268391s (0.000000117s flush + 0.000268274s kv commit) 2019-12-07T19:34:04.684+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.684+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.684+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e72000> 2019-12-07T19:34:04.684+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.684+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 kv_submitted 2019-12-07T19:34:04.684+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72000 2019-12-07T19:34:04.684+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 finishing 2019-12-07T19:34:04.684+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72000 onodes 2019-12-07T19:34:04.684+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72000 done 2019-12-07T19:34:04.684+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.684+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72000 [] 2019-12-07T19:34:04.684+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.684+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.684+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.684+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_update_log_missing_reply(1.8s0 epoch 27/26 rep_tid 115 lcod 21'3) v3 -- 0x558e81964a00 con 0x558e818d7c00 2019-12-07T19:34:04.684+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.687+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 153 ==== MRecoveryReserve(1.7s3 RELEASE e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e80231e00 con 0x558e818d7c00 2019-12-07T19:34:04.687+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.7s3 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.687+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.687+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.687+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558e8122e000 2019-12-07T19:34:04.687+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.687+0000 7fe2c8267700 5 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.687820+0000 1 0.000040 2019-12-07T19:34:04.687+0000 7fe2c8267700 5 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.687+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:04.687+0000 7fe2cea74700 15 osd.0 27 enqueue_peering_evt 1.bs1 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.687+0000 7fe2cea74700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.687+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.687+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.687+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e83c78000 2019-12-07T19:34:04.687+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.687+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MRecoveryReserve(1.bs0 GRANT e27/27) v3 -- 0x558e84198000 con 0x558e818fe400 2019-12-07T19:34:04.687+0000 7fe2ca26b700 5 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.687897+0000 1 0.000022 2019-12-07T19:34:04.687+0000 7fe2ca26b700 5 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.687+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:04.687+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.687+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 154 ==== MRecoveryReserve(1.8s5 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e80231e00 con 0x558e818d7c00 2019-12-07T19:34:04.687+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.8s5 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.687+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.687+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.687+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558e841a4000 2019-12-07T19:34:04.687+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'19 lc 21'3 (0'0,27'19] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=6 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.687+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 81 ==== MRecoveryReserve(1.2s0 GRANT e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e819b05a0 con 0x558e818fe400 2019-12-07T19:34:04.687+0000 7fe2e6d3a700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.687+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.687+0000 7fe2c7a66700 5 osd.0 pg_epoch: 27 pg[1.8s5( v 27'19 lc 21'3 (0'0,27'19] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=6 mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.687989+0000 3 0.000266 2019-12-07T19:34:04.687+0000 7fe2c7a66700 5 osd.0 pg_epoch: 27 pg[1.8s5( v 27'19 lc 21'3 (0'0,27'19] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=6 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.687+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:04.687+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.687+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.687+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.687+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.687+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_recovery_priority recovery priority is 190 2019-12-07T19:34:04.687+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MRecoveryReserve(1.2s3 REQUEST prio: 190 e27/27) v3 -- 0x558e819c5a40 con 0x558e818d6800 2019-12-07T19:34:04.687+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.687+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.687+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 35 ==== MRecoveryReserve(1.2s0 GRANT e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e819c5a40 con 0x558e818d6800 2019-12-07T19:34:04.687+0000 7fe2e753b700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.687+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.687+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.687+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.687+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.687+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_recovery_priority recovery priority is 190 2019-12-07T19:34:04.687+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MRecoveryReserve(1.2s4 REQUEST prio: 190 e27/27) v3 -- 0x558e819b14a0 con 0x558e818fe000 2019-12-07T19:34:04.687+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.687+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.687+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 20 ==== MRecoveryReserve(1.2s0 GRANT e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e819b14a0 con 0x558e818fe000 2019-12-07T19:34:04.687+0000 7fe2e753b700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.687+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.687+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.687+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.687+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.687+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.688+0000 7fe2caa6c700 5 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] exit Started/Primary/Active/WaitRemoteRecoveryReserved 2019-12-07T19:34:04.688698+0000 5 0.000152 2019-12-07T19:34:04.688+0000 7fe2caa6c700 5 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovery_wait+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] enter Started/Primary/Active/Recovering 2019-12-07T19:34:04.688+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] queue_recovery -- queuing 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) 2019-12-07T19:34:04.688+0000 7fe2caa6c700 10 osd.0 27 _maybe_queue_recovery starting 1, recovery_ops_reserved 0 -> 1 2019-12-07T19:34:04.688+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.688+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) queued 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(4)=3} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0 ml 3 missing shards 0 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1 ml 3 missing shards 0 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 2 ml 3 missing shards 0 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3 ml 3 missing shards 1 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4 ml 3 missing shards 1 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5 ml 3 missing shards 0 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing based degraded 3 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing based misplaced 3 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 3 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 3 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 27:52 2019-12-07T19:34:04.688+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.688+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.688+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) pg 0x558e81075000 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 27 do_recovery starting 1 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recover_replicas(1) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] peer osd.5(3) missing 4 objects. 2019-12-07T19:34:04.688+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] peer osd.5(3) missing {1:41df217a:::smithi13140796-39:head=21'5 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:41fe8a11:::smithi13140796-45:head=21'6 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4986b721:::smithi13140796-43:head=21'4 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recover_replicas: recover_object_replicas(1:4986b721:::smithi13140796-43:head) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prep_object_replica_pushes: on 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_object_context: obc NOT found in cache: 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.688+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# _ 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# _ = 0 2019-12-07T19:34:04.688+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# snapset 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# snapset = 0 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] populate_obc_watchers 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.688+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.688+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattrs 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattrs 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# = 0 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_object_context: creating obc from disk: 0x558e83ff0000 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_object_context: 0x558e83ff0000 1:4986b721:::smithi13140796-43:head rwstate(none n=0 w=0) oi: 1:4986b721:::smithi13140796-43:head(21'4 client.4540.0:128 dirty|data_digest s 720896 uv 4 dd 2bc8c113 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e81123c20 snapset: 0=[]:{} 2019-12-07T19:34:04.688+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recovery got recovery read lock on 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] start_recovery_op 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 27 start_recovery_op pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 1:4986b721:::smithi13140796-43:head (0/10 rops) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] checking 0(0) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] checking 1(2) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] checking 3(5) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] checking 4(1) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] checking 5(3) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] checking 7(4) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] recover_object: built op RecoveryOp(hoid=1:4986b721:::smithi13140796-43:head v=21'4 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=5 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] peer osd.7(4) missing 4 objects. 2019-12-07T19:34:04.688+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] peer osd.7(4) missing {1:41df217a:::smithi13140796-39:head=21'5 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:41fe8a11:::smithi13140796-45:head=21'6 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4986b721:::smithi13140796-43:head=21'4 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] run_recovery_op: starting RecoveryOp(hoid=1:4986b721:::smithi13140796-43:head v=21'4 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:4986b721:::smithi13140796-43:head v=21'4 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=4 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_hash_info: Getting attr on 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_hash_info: not in cache 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_hash_info: found on disk, size 180224 2019-12-07T19:34:04.688+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# hinfo_key = 0 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 0(0) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking acting 4(1) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking missing_loc 0(0) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking missing_loc 1(2) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking missing_loc 3(5) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking missing_loc 4(1) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_all_avail_shards: checking missing_loc 5(4) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] continue_recovery_op: IDLE return RecoveryOp(hoid=1:4986b721:::smithi13140796-43:head v=21'4 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=4 state=READING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] start_read_op: starting ReadOp(tid=20, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_read_op: starting read ReadOp(tid=20, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=20, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e84242500 con 0x558e802a5400 2019-12-07T19:34:04.688+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.2s2 27/26 ECSubRead(tid=20, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e84243b80 con 0x558e818d7c00 2019-12-07T19:34:04.688+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.2s1 27/26 ECSubRead(tid=20, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e84243900 con 0x558e818fe400 2019-12-07T19:34:04.688+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=20, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558e84242500 con 0x558e802a5400 2019-12-07T19:34:04.688+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.2s4 27/26 ECSubRead(tid=20, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e84243680 con 0x558e818d6800 2019-12-07T19:34:04.688+0000 7fe2d427f700 15 osd.0 27 enqueue_op 0x558e83edcdc0 prio 3 cost 0 latency 0.000032 epoch 27 MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=20, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.688+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=20, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=20, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] do_read_op: started ReadOp(tid=20, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={1:4986b721:::smithi13140796-43:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:4986b721:::smithi13140796-43:head,1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(4)=1:4986b721:::smithi13140796-43:head}, in_progress=0(0),1(2),4(1),5(4)) 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] started 1 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 27 do_recovery started 1/1 on pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.688+0000 7fe2c6a64700 10 osd.0 27 release_reserved_pushes(1), recovery_ops_reserved 1 -> 0 2019-12-07T19:34:04.688+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=20, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edcdc0 prio 3 cost 0 latency 0.000094 MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=20, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=20, to_read={1:4986b721:::smithi13140796-43:head=0,262144,0}, subchunks={1:4986b721:::smithi13140796-43:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.688+0000 7fe2caa6c700 25 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.688+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 0x0~40000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~40000 size 0x2c000 (180224) 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff2850 blob([0x1ea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff27e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff28c0 blob([0x1ec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2930 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff2e00 blob([0x1ee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2cb0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff2e70 blob([0x1f0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2d20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff2ee0 blob([0x1f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2d90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff30a0 blob([0x1f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2f50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff3110 blob([0x1f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2fc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff3180 blob([0x1f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3030 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff31f0 blob([0x1fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3260 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff32d0 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3340 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff33b0 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3420 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff3490 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3500 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff3570 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff35e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff3650 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff36c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff3730 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff37a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff3810 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3880 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff38f0 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3960 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff39d0 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3a40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff3ab0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3b20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff3b90 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3c00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff3c70 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3ce0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff3d50 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3dc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff2850 blob([0x1ea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff27e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff28c0 blob([0x1ec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2930 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff2e00 blob([0x1ee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2cb0 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff2e70 blob([0x1f0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2d20 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff2ee0 blob([0x1f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2d90 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff30a0 blob([0x1f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2f50 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff3110 blob([0x1f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2fc0 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff3180 blob([0x1f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3030 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff31f0 blob([0x1fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3260 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff32d0 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3340 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff33b0 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3420 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff3490 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3500 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff3570 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff35e0 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff3650 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff36c0 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff3730 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff37a0 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff3810 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3880 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff38f0 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3960 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff39d0 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3a40 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff3ab0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3b20 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff3b90 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3c00 sbid 0x0)) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff3c70 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3ce0 sbid 0x0)) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ff3d50 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3dc0 sbid 0x0)) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff2850 blob([0x1ea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff27e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff27f8 in 0x558e802195e0) _discard 0x1000~1000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83c8fb90 space 0x558e83ff27f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff28c0 blob([0x1ec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2930 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff2948 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e840321b0 space 0x558e83ff2948 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff2e00 blob([0x1ee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2cb0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff2cc8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e840339e0 space 0x558e83ff2cc8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff2e70 blob([0x1f0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2d20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff2d38 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e81170480 space 0x558e83ff2d38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff2ee0 blob([0x1f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2d90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff2da8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83d119e0 space 0x558e83ff2da8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff30a0 blob([0x1f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2f50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff2f68 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83d102d0 space 0x558e83ff2f68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff3110 blob([0x1f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff2fc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff2fd8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83d10240 space 0x558e83ff2fd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff3180 blob([0x1f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3030 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3048 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e84032b40 space 0x558e83ff3048 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff31f0 blob([0x1fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3260 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3278 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74000 space 0x558e83ff3278 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff32d0 blob([0x1fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3340 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3358 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74090 space 0x558e83ff3358 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff33b0 blob([0x1fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3420 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3438 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74120 space 0x558e83ff3438 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff3490 blob([0x200000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3500 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3518 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e741b0 space 0x558e83ff3518 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff3570 blob([0x202000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff35e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff35f8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74240 space 0x558e83ff35f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff3650 blob([0x204000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff36c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff36d8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e742d0 space 0x558e83ff36d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff3730 blob([0x206000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff37a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff37b8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74360 space 0x558e83ff37b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff3810 blob([0x208000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3880 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3898 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e743f0 space 0x558e83ff3898 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff38f0 blob([0x20a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3960 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3978 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74480 space 0x558e83ff3978 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff39d0 blob([0x20c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3a40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3a58 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74510 space 0x558e83ff3a58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff3ab0 blob([0x20e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3b20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3b38 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e745a0 space 0x558e83ff3b38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff3b90 blob([0x210000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3c00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3c18 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74630 space 0x558e83ff3c18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff3c70 blob([0x212000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3ce0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3cf8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e746c0 space 0x558e83ff3cf8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ff3d50 blob([0x214000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff3dc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83ff3dd8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74750 space 0x558e83ff3dd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.689+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:4986b721:::smithi13140796-43:head# 0x0~40000 = 180224 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read read request=262144 r=180224 len=180224 2019-12-07T19:34:04.689+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] get_hash_info: Getting attr on 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read: Checking hash of 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.689+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 -- 0x558e8417c000 con 0x558e802a5400 2019-12-07T19:34:04.689+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edcdc0 finish 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.689+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558e8417c000 con 0x558e802a5400 2019-12-07T19:34:04.689+0000 7fe2d427f700 15 osd.0 27 enqueue_op 0x558e84176580 prio 127 cost 0 latency 0.000017 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 2019-12-07T19:34:04.689+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.689+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.689+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.689+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.689+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.689+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e84176580 prio 127 cost 0 latency 0.000037 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 2019-12-07T19:34:04.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=20, attrs_read=0) 2019-12-07T19:34:04.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=20, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [0(0),180224]))}, priority=3, obj_to_source={1:4986b721:::smithi13140796-43:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:4986b721:::smithi13140796-43:head,1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(4)=1:4986b721:::smithi13140796-43:head}, in_progress=1(2),4(1),5(4)) 2019-12-07T19:34:04.689+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e84176580 finish 2019-12-07T19:34:04.689+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.689+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 155 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 ==== 180389+0+0 (unknown 0 0 0) 0x558e84243b80 con 0x558e818d7c00 2019-12-07T19:34:04.689+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edc9a0 prio 127 cost 0 latency 0.000014 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 2019-12-07T19:34:04.689+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edc9a0 prio 127 cost 0 latency 0.000033 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.689+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 2019-12-07T19:34:04.689+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=20, attrs_read=0) 2019-12-07T19:34:04.689+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 82 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 ==== 180389+0+0 (unknown 0 0 0) 0x558e84243900 con 0x558e818fe400 2019-12-07T19:34:04.689+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e81942c60 prio 127 cost 0 latency 0.000016 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 2019-12-07T19:34:04.689+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.689+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=20, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [0(0),180224, 1(2),180224]))}, priority=3, obj_to_source={1:4986b721:::smithi13140796-43:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:4986b721:::smithi13140796-43:head,1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(4)=1:4986b721:::smithi13140796-43:head}, in_progress=4(1),5(4)) 2019-12-07T19:34:04.689+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edc9a0 finish 2019-12-07T19:34:04.689+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.689+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.689+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.689+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.689+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.689+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e81942c60 prio 127 cost 0 latency 0.000043 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 2019-12-07T19:34:04.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=20, attrs_read=0) 2019-12-07T19:34:04.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=20, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [0(0),180224, 1(2),180224, 4(1),180224]))}, priority=3, obj_to_source={1:4986b721:::smithi13140796-43:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:4986b721:::smithi13140796-43:head,1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(4)=1:4986b721:::smithi13140796-43:head}, in_progress=5(4)) 2019-12-07T19:34:04.689+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e81942c60 finish 2019-12-07T19:34:04.689+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.690+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 36 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 ==== 180389+0+0 (unknown 0 0 0) 0x558e84243680 con 0x558e818d6800 2019-12-07T19:34:04.690+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e83b0b600 prio 127 cost 0 latency 0.000018 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 2019-12-07T19:34:04.690+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.690+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.690+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.690+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.690+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.690+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83b0b600 prio 127 cost 0 latency 0.000043 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.690+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=20, attrs_read=0)) v2 2019-12-07T19:34:04.690+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=20, attrs_read=0) 2019-12-07T19:34:04.690+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:04.690+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:04.690+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=1 2019-12-07T19:34:04.690+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply have shard=4 2019-12-07T19:34:04.690+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_sub_read_reply Complete: ReadOp(tid=20, to_read={1:4986b721:::smithi13140796-43:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:4986b721:::smithi13140796-43:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [0(0),180224, 1(2),180224, 4(1),180224, 5(4),180224]))}, priority=3, obj_to_source={1:4986b721:::smithi13140796-43:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:4986b721:::smithi13140796-43:head,1(2)=1:4986b721:::smithi13140796-43:head,4(1)=1:4986b721:::smithi13140796-43:head,5(4)=1:4986b721:::smithi13140796-43:head}, in_progress=) 2019-12-07T19:34:04.690+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_recovery_read_complete: returned 1:4986b721:::smithi13140796-43:head (0, 1048576, [0(0),180224, 1(2),180224, 4(1),180224, 5(4),180224]) 2019-12-07T19:34:04.690+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] handle_recovery_read_complete: [0,180224, 1,180224, 2,180224, 4,180224] 2019-12-07T19:34:04.691+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:4986b721:::smithi13140796-43:head v=21'4 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=READING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:04.691+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] continue_recovery_op: before_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), after_progress=ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), pop.data.length()=180224, size=720896 2019-12-07T19:34:04.691+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] continue_recovery_op: before_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), after_progress=ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), pop.data.length()=180224, size=720896 2019-12-07T19:34:04.691+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] continue_recovery_op: READING return RecoveryOp(hoid=1:4986b721:::smithi13140796-43:head v=21'4 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=WRITING waiting_on_pushes=5(3),7(4) extent_requested=0,1048576) 2019-12-07T19:34:04.691+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDPGPush(1.2s3 27/26 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 -- 0x558e84029200 con 0x558e818d6800 2019-12-07T19:34:04.691+0000 7fe2caa6c700 10 osd.0 27 send_incremental_map 26 -> 27 to 0x558e818fe000 v2:172.21.15.109:6826/36939 2019-12-07T19:34:04.691+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(27..27 src has 1..27) v4 -- 0x558e8419d680 con 0x558e818fe000 2019-12-07T19:34:04.691+0000 7fe2caa6c700 10 osd.0 27 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 26 -> 27 (shared) 2019-12-07T19:34:04.691+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDPGPush(1.2s4 27/26 [PushOp(1:4986b721:::smithi13140796-43:head, version: 21'4, data_included: [0~180224], data_size: 180224, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 -- 0x558e84028800 con 0x558e818fe000 2019-12-07T19:34:04.691+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83b0b600 finish 2019-12-07T19:34:04.691+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.693+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 21 ==== MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3 ==== 111+0+0 (unknown 0 0 0) 0x558e84028800 con 0x558e818fe000 2019-12-07T19:34:04.693+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e81943b80 prio 127 cost 1049576 latency 0.000020 epoch 27 MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3 2019-12-07T19:34:04.693+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3) prio 127 cost 1049576 e27) 2019-12-07T19:34:04.693+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.693+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3) prio 127 cost 1049576 e27) queued 2019-12-07T19:34:04.693+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.693+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3) prio 127 cost 1049576 e27) pg 0x558e81075000 2019-12-07T19:34:04.693+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e81943b80 prio 127 cost 1049576 latency 0.000054 MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.693+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3 2019-12-07T19:34:04.693+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:4986b721:::smithi13140796-43:head v=21'4 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=WRITING waiting_on_pushes=5(3) extent_requested=0,1048576) 2019-12-07T19:34:04.693+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e81943b80 finish 2019-12-07T19:34:04.693+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.693+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 37 ==== MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3 ==== 111+0+0 (unknown 0 0 0) 0x558e84053000 con 0x558e818d6800 2019-12-07T19:34:04.693+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e81942160 prio 127 cost 1049576 latency 0.000013 epoch 27 MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3 2019-12-07T19:34:04.693+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3) prio 127 cost 1049576 e27) 2019-12-07T19:34:04.693+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.693+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3) prio 127 cost 1049576 e27) queued 2019-12-07T19:34:04.693+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.693+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3) prio 127 cost 1049576 e27) pg 0x558e81075000 2019-12-07T19:34:04.693+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81942160 prio 127 cost 1049576 latency 0.000045 MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] 2019-12-07T19:34:04.693+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] _handle_message: MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:4986b721:::smithi13140796-43:head)]) v3 2019-12-07T19:34:04.693+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:4986b721:::smithi13140796-43:head v=21'4 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=WRITING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:04.693+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] continue_recovery_op: on_peer_recover on 5(3), obj 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.693+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.693+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.693+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.693+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.693+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 4 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(4)=3} 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0 ml 3 missing shards 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1 ml 3 missing shards 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 2 ml 3 missing shards 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3 ml 3 missing shards 1 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4 ml 3 missing shards 1 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5 ml 3 missing shards 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing based degraded 3 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing based misplaced 3 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 3 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 3 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.694+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=3},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 27:53 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] continue_recovery_op: on_peer_recover on 7(4), obj 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 3 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 4 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(3),5(4)=1,0(0),1(2),3(5),4(1),5(4)=2} 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 0 ml 3 missing shards 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 1 ml 3 missing shards 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 2 ml 3 missing shards 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3 ml 2 missing shards 1 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 3 ml 1 missing shards 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 4 ml 3 missing shards 1 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats shard 5 ml 3 missing shards 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing based degraded 2 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats missing based misplaced 3 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats degraded 2 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] update_calc_stats misplaced 3 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.694+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=3},1={(1+0)=3},2={(1+0)=3},3={(0+0)=2,(1+0)=1},4={(0+1)=3},5={(1+0)=3}} ps=[2~1]] publish_stats_to_osd 27:54 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 3 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 3 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1)=1,0(0),1(2),3(5),4(1),5(4)=2} 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 0 ml 2 missing shards 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 1 ml 2 missing shards 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 2 ml 2 missing shards 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 3 ml 2 missing shards 1 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 4 ml 2 missing shards 1 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 5 ml 2 missing shards 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats missing based degraded 2 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats missing based misplaced 2 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats degraded 2 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats misplaced 2 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.694+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] publish_stats_to_osd 27:55 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] pushed 1:4986b721:::smithi13140796-43:head to all replicas 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] finish_recovery_op 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 27 finish_recovery_op pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] 1:4986b721:::smithi13140796-43:head dequeue=0 (1/10 rops) 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] queue_recovery -- queuing 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 27 _maybe_queue_recovery starting 1, recovery_ops_reserved 0 -> 1 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] release_backoffs [1:4986b721:::smithi13140796-43:head,1:4986b721:::smithi13140796-43:head) 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] finish_degraded_object 1:4986b721:::smithi13140796-43:head 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] continue_recovery_op: WRITING return RecoveryOp(hoid=1:4986b721:::smithi13140796-43:head v=21'4 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:4986b721:::smithi13140796-43:head@21'4, size: 720896, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:720896, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=2 state=COMPLETE waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81942160 finish 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) queued 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) pg 0x558e81075000 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 27 do_recovery starting 1 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] recover_replicas(1) 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] peer osd.5(3) missing 3 objects. 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] peer osd.5(3) missing {1:41df217a:::smithi13140796-39:head=21'5 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:41fe8a11:::smithi13140796-45:head=21'6 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] recover_replicas: recover_object_replicas(1:41df217a:::smithi13140796-39:head) 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] prep_object_replica_pushes: on 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_object_context: obc NOT found in cache: 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.694+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# _ 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# _ = 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# snapset 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# snapset = 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] populate_obc_watchers 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.694+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattrs 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattrs 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# = 0 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_object_context: creating obc from disk: 0x558e8405d200 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_object_context: 0x558e8405d200 1:41df217a:::smithi13140796-39:head rwstate(none n=0 w=0) oi: 1:41df217a:::smithi13140796-39:head(21'5 client.4540.0:116 dirty|data_digest s 638976 uv 5 dd b038798b alloc_hint [0 0 0]) exists: 1 ssc: 0x558e83dccb40 snapset: 0=[]:{} 2019-12-07T19:34:04.694+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] recovery got recovery read lock on 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] start_recovery_op 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 27 start_recovery_op pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] 1:41df217a:::smithi13140796-39:head (0/10 rops) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] checking 0(0) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] checking 1(2) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] checking 3(5) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] checking 4(1) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] checking 5(3) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] checking 7(4) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] recover_object: built op RecoveryOp(hoid=1:41df217a:::smithi13140796-39:head v=21'5 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=5 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] peer osd.7(4) missing 3 objects. 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] peer osd.7(4) missing {1:41df217a:::smithi13140796-39:head=21'5 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:41fe8a11:::smithi13140796-45:head=21'6 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] run_recovery_op: starting RecoveryOp(hoid=1:41df217a:::smithi13140796-39:head v=21'5 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:41df217a:::smithi13140796-39:head v=21'5 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=4 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_hash_info: Getting attr on 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_hash_info: not in cache 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_hash_info: found on disk, size 159744 2019-12-07T19:34:04.695+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key = 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_all_avail_shards: checking acting 0(0) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_all_avail_shards: checking acting 4(1) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_all_avail_shards: checking missing_loc 0(0) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_all_avail_shards: checking missing_loc 1(2) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_all_avail_shards: checking missing_loc 3(5) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_all_avail_shards: checking missing_loc 4(1) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_all_avail_shards: checking missing_loc 5(4) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] continue_recovery_op: IDLE return RecoveryOp(hoid=1:41df217a:::smithi13140796-39:head v=21'5 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=4 state=READING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] start_read_op: starting ReadOp(tid=21, to_read={1:41df217a:::smithi13140796-39:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:41df217a:::smithi13140796-39:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] do_read_op: starting read ReadOp(tid=21, to_read={1:41df217a:::smithi13140796-39:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:41df217a:::smithi13140796-39:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=21, to_read={1:41df217a:::smithi13140796-39:head=0,262144,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e84243680 con 0x558e802a5400 2019-12-07T19:34:04.695+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.2s2 27/26 ECSubRead(tid=21, to_read={1:41df217a:::smithi13140796-39:head=0,262144,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e84243b80 con 0x558e818d7c00 2019-12-07T19:34:04.695+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=21, to_read={1:41df217a:::smithi13140796-39:head=0,262144,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558e84243680 con 0x558e802a5400 2019-12-07T19:34:04.695+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.2s1 27/26 ECSubRead(tid=21, to_read={1:41df217a:::smithi13140796-39:head=0,262144,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e84242500 con 0x558e818fe400 2019-12-07T19:34:04.695+0000 7fe2d427f700 15 osd.0 27 enqueue_op 0x558e841766e0 prio 3 cost 0 latency 0.000018 epoch 27 MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=21, to_read={1:41df217a:::smithi13140796-39:head=0,262144,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.695+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=21, to_read={1:41df217a:::smithi13140796-39:head=0,262144,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.2s4 27/26 ECSubRead(tid=21, to_read={1:41df217a:::smithi13140796-39:head=0,262144,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e84242280 con 0x558e818d6800 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] do_read_op: started ReadOp(tid=21, to_read={1:41df217a:::smithi13140796-39:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:41df217a:::smithi13140796-39:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={1:41df217a:::smithi13140796-39:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:41df217a:::smithi13140796-39:head,1(2)=1:41df217a:::smithi13140796-39:head,4(1)=1:41df217a:::smithi13140796-39:head,5(4)=1:41df217a:::smithi13140796-39:head}, in_progress=0(0),1(2),4(1),5(4)) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] started 1 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 27 do_recovery started 1/1 on pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 27 release_reserved_pushes(1), recovery_ops_reserved 1 -> 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=21, to_read={1:41df217a:::smithi13140796-39:head=0,262144,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=21, to_read={1:41df217a:::smithi13140796-39:head=0,262144,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e841766e0 prio 3 cost 0 latency 0.000099 MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=21, to_read={1:41df217a:::smithi13140796-39:head=0,262144,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=21, to_read={1:41df217a:::smithi13140796-39:head=0,262144,0}, subchunks={1:41df217a:::smithi13140796-39:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.695+0000 7fe2caa6c700 25 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.695+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 0x0~40000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~40000 size 0x27000 (159744) 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd2ee0 blob([0x1c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e70 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd2bd0 blob([0x1c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd30a0 blob([0x1c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3180 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd2fc0 blob([0x1c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2c40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd3030 blob([0x1cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2b60 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd3110 blob([0x1cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2af0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd23f0 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2380 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd2310 blob([0x1d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd22a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd2a80 blob([0x1d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd29a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd3260 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd32d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd3340 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd33b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd3420 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3490 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd3500 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3570 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd35e0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3650 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd36c0 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3730 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd37a0 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3810 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd3880 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd38f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd3960 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd39d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd3a40 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3ab0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd3b20 blob([0x1e9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83dd3b90 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd2310 blob([0x1d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd22a0 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd23f0 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2380 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd2a80 blob([0x1d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd29a0 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd2bd0 blob([0x1c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e00 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd2ee0 blob([0x1c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e70 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd2fc0 blob([0x1c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2c40 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd3030 blob([0x1cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2b60 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd30a0 blob([0x1c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3180 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd3110 blob([0x1cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2af0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd3260 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd32d0 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd3340 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd33b0 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd3420 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3490 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd3500 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3570 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd35e0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3650 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd36c0 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3730 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd37a0 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3810 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd3880 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd38f0 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd3960 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd39d0 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd3a40 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3ab0 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd3b20 blob([0x1e9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83dd3b90 sbid 0x0)) need {<0x0, 0x1000> : [0x26000:0~1000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.695+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd2310 blob([0x1d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd22a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd22b8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83d11830 space 0x558e83dd22b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd23f0 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2380 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd2398 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e747e0 space 0x558e83dd2398 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd2a80 blob([0x1d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd29a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd29b8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74870 space 0x558e83dd29b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd2bd0 blob([0x1c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd2e18 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74900 space 0x558e83dd2e18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd2ee0 blob([0x1c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e70 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd2e88 in 0x558e802195e0) _discard 0x1000~1000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74990 space 0x558e83dd2e88 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd2fc0 blob([0x1c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2c40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd2c58 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74a20 space 0x558e83dd2c58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd3030 blob([0x1cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2b60 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd2b78 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74ab0 space 0x558e83dd2b78 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd30a0 blob([0x1c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3180 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3198 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74b40 space 0x558e83dd3198 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd3110 blob([0x1cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2af0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd2b08 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74bd0 space 0x558e83dd2b08 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd3260 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd32d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd32e8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74c60 space 0x558e83dd32e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd3340 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd33b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd33c8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74cf0 space 0x558e83dd33c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd3420 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3490 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd34a8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74d80 space 0x558e83dd34a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd3500 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3570 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3588 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74e10 space 0x558e83dd3588 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd35e0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3650 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3668 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74ea0 space 0x558e83dd3668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd36c0 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3730 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3748 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74f30 space 0x558e83dd3748 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd37a0 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3810 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3828 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e74fc0 space 0x558e83dd3828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd3880 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd38f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3908 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e75050 space 0x558e83dd3908 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd3960 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd39d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd39e8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e750e0 space 0x558e83dd39e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd3a40 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3ab0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3ac8 in 0x558e802195e0) _discard 0x0~2000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e75170 space 0x558e83dd3ac8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd3b20 blob([0x1e9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83dd3b90 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x26000:0~1000]} 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 bluestore.BufferSpace(0x558e83dd3ba8 in 0x558e802195e0) _discard 0x0~1000 2019-12-07T19:34:04.695+0000 7fe2caa6c700 20 _add level 1 near 0 on buffer(0x558e83e75200 space 0x558e83dd3ba8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.696+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 0x0~40000 = 159744 2019-12-07T19:34:04.696+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read read request=262144 r=159744 len=159744 2019-12-07T19:34:04.696+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] get_hash_info: Getting attr on 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.696+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read: Checking hash of 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.696+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 -- 0x558e84242780 con 0x558e802a5400 2019-12-07T19:34:04.696+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e841766e0 finish 2019-12-07T19:34:04.696+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.696+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558e84242780 con 0x558e802a5400 2019-12-07T19:34:04.696+0000 7fe2d427f700 15 osd.0 27 enqueue_op 0x558e84176840 prio 127 cost 0 latency 0.000017 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 2019-12-07T19:34:04.696+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.696+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.696+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.696+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.696+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.696+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e84176840 prio 127 cost 0 latency 0.000034 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] 2019-12-07T19:34:04.696+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 2019-12-07T19:34:04.696+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=21, attrs_read=0) 2019-12-07T19:34:04.696+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=21, to_read={1:41df217a:::smithi13140796-39:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:41df217a:::smithi13140796-39:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [0(0),159744]))}, priority=3, obj_to_source={1:41df217a:::smithi13140796-39:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:41df217a:::smithi13140796-39:head,1(2)=1:41df217a:::smithi13140796-39:head,4(1)=1:41df217a:::smithi13140796-39:head,5(4)=1:41df217a:::smithi13140796-39:head}, in_progress=1(2),4(1),5(4)) 2019-12-07T19:34:04.696+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e84176840 finish 2019-12-07T19:34:04.696+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.696+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 83 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 ==== 159909+0+0 (unknown 0 0 0) 0x558e84242500 con 0x558e818fe400 2019-12-07T19:34:04.696+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e83edd080 prio 127 cost 0 latency 0.000019 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 2019-12-07T19:34:04.696+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.696+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 156 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 ==== 159909+0+0 (unknown 0 0 0) 0x558e84243b80 con 0x558e818d7c00 2019-12-07T19:34:04.696+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edc2c0 prio 127 cost 0 latency 0.000012 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 2019-12-07T19:34:04.696+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.696+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 38 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 ==== 159909+0+0 (unknown 0 0 0) 0x558e84242280 con 0x558e818d6800 2019-12-07T19:34:04.696+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e81943340 prio 127 cost 0 latency 0.000017 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 2019-12-07T19:34:04.696+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edd080 prio 127 cost 0 latency 0.000607 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=21, attrs_read=0) 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=21, to_read={1:41df217a:::smithi13140796-39:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:41df217a:::smithi13140796-39:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [0(0),159744, 4(1),159744]))}, priority=3, obj_to_source={1:41df217a:::smithi13140796-39:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:41df217a:::smithi13140796-39:head,1(2)=1:41df217a:::smithi13140796-39:head,4(1)=1:41df217a:::smithi13140796-39:head,5(4)=1:41df217a:::smithi13140796-39:head}, in_progress=1(2),5(4)) 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edd080 finish 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81943340 prio 127 cost 0 latency 0.000369 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=21, attrs_read=0) 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=21, to_read={1:41df217a:::smithi13140796-39:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:41df217a:::smithi13140796-39:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [0(0),159744, 4(1),159744, 5(4),159744]))}, priority=3, obj_to_source={1:41df217a:::smithi13140796-39:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:41df217a:::smithi13140796-39:head,1(2)=1:41df217a:::smithi13140796-39:head,4(1)=1:41df217a:::smithi13140796-39:head,5(4)=1:41df217a:::smithi13140796-39:head}, in_progress=1(2)) 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81943340 finish 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edc2c0 prio 127 cost 0 latency 0.000650 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=21, attrs_read=0)) v2 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=21, attrs_read=0) 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read_reply have shard=1 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read_reply have shard=4 2019-12-07T19:34:04.697+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_sub_read_reply Complete: ReadOp(tid=21, to_read={1:41df217a:::smithi13140796-39:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:41df217a:::smithi13140796-39:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [0(0),159744, 1(2),159744, 4(1),159744, 5(4),159744]))}, priority=3, obj_to_source={1:41df217a:::smithi13140796-39:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:41df217a:::smithi13140796-39:head,1(2)=1:41df217a:::smithi13140796-39:head,4(1)=1:41df217a:::smithi13140796-39:head,5(4)=1:41df217a:::smithi13140796-39:head}, in_progress=) 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_recovery_read_complete: returned 1:41df217a:::smithi13140796-39:head (0, 1048576, [0(0),159744, 1(2),159744, 4(1),159744, 5(4),159744]) 2019-12-07T19:34:04.697+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] handle_recovery_read_complete: [0,159744, 1,159744, 2,159744, 4,159744] 2019-12-07T19:34:04.698+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:41df217a:::smithi13140796-39:head v=21'5 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=READING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:04.698+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] continue_recovery_op: before_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), after_progress=ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), pop.data.length()=159744, size=638976 2019-12-07T19:34:04.698+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] continue_recovery_op: before_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), after_progress=ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), pop.data.length()=159744, size=638976 2019-12-07T19:34:04.698+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] continue_recovery_op: READING return RecoveryOp(hoid=1:41df217a:::smithi13140796-39:head v=21'5 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=WRITING waiting_on_pushes=5(3),7(4) extent_requested=0,1048576) 2019-12-07T19:34:04.698+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDPGPush(1.2s3 27/26 [PushOp(1:41df217a:::smithi13140796-39:head, version: 21'5, data_included: [0~159744], data_size: 159744, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 -- 0x558e84053000 con 0x558e818d6800 2019-12-07T19:34:04.698+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDPGPush(1.2s4 27/26 [PushOp(1:41df217a:::smithi13140796-39:head, version: 21'5, data_included: [0~159744], data_size: 159744, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 -- 0x558e84197600 con 0x558e818fe000 2019-12-07T19:34:04.698+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edc2c0 finish 2019-12-07T19:34:04.698+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.700+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 39 ==== MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3 ==== 111+0+0 (unknown 0 0 0) 0x558e84197600 con 0x558e818d6800 2019-12-07T19:34:04.700+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e81942f20 prio 127 cost 1049576 latency 0.000016 epoch 27 MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3 2019-12-07T19:34:04.700+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3) prio 127 cost 1049576 e27) 2019-12-07T19:34:04.700+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.700+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3) prio 127 cost 1049576 e27) queued 2019-12-07T19:34:04.700+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.700+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3) prio 127 cost 1049576 e27) pg 0x558e81075000 2019-12-07T19:34:04.700+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e81942f20 prio 127 cost 1049576 latency 0.000043 MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] 2019-12-07T19:34:04.700+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 22 ==== MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3 ==== 111+0+0 (unknown 0 0 0) 0x558e81955600 con 0x558e818fe000 2019-12-07T19:34:04.700+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e818051e0 prio 127 cost 1049576 latency 0.000011 epoch 27 MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3 2019-12-07T19:34:04.700+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3) prio 127 cost 1049576 e27) 2019-12-07T19:34:04.700+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] _handle_message: MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3 2019-12-07T19:34:04.700+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:41df217a:::smithi13140796-39:head v=21'5 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=WRITING waiting_on_pushes=7(4) extent_requested=0,1048576) 2019-12-07T19:34:04.700+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e81942f20 finish 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3) prio 127 cost 1049576 e27) queued 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3) prio 127 cost 1049576 e27) pg 0x558e81075000 2019-12-07T19:34:04.700+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.700+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e818051e0 prio 127 cost 1049576 latency 0.000050 MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] 2019-12-07T19:34:04.700+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] _handle_message: MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41df217a:::smithi13140796-39:head)]) v3 2019-12-07T19:34:04.700+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:41df217a:::smithi13140796-39:head v=21'5 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=WRITING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:04.700+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] continue_recovery_op: on_peer_recover on 5(3), obj 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 3 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 3 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1)=1,0(0),1(2),3(5),4(1),5(4)=2} 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 0 ml 2 missing shards 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 1 ml 2 missing shards 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 2 ml 2 missing shards 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 3 ml 2 missing shards 1 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 4 ml 2 missing shards 1 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 5 ml 2 missing shards 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats missing based degraded 2 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats missing based misplaced 2 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats degraded 2 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats misplaced 2 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.700+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=2},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] publish_stats_to_osd 27:56 2019-12-07T19:34:04.700+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] continue_recovery_op: on_peer_recover on 7(4), obj 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 2 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 3 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1)=1,0(0),1(2),3(5),4(1),5(3),5(4)=1,0(0),1(2),3(5),4(1),5(4)=1} 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 0 ml 2 missing shards 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 1 ml 2 missing shards 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 2 ml 2 missing shards 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 3 ml 1 missing shards 1 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 3 ml 1 missing shards 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 4 ml 2 missing shards 1 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats shard 5 ml 2 missing shards 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats missing based degraded 1 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats missing based misplaced 2 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats degraded 1 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] update_calc_stats misplaced 2 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.700+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=2},1={(1+0)=2},2={(1+0)=2},3={(0+0)=1,(1+0)=1},4={(0+1)=2},5={(1+0)=2}} ps=[2~1]] publish_stats_to_osd 27:57 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 2 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 2 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1)=2,0(0),1(2),3(5),4(1),5(4)=1} 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 0 ml 1 missing shards 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 1 ml 1 missing shards 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 2 ml 1 missing shards 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 3 ml 1 missing shards 1 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 4 ml 1 missing shards 1 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 5 ml 1 missing shards 0 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats missing based degraded 1 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats missing based misplaced 1 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats degraded 1 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats misplaced 1 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.700+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] publish_stats_to_osd 27:58 2019-12-07T19:34:04.700+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] pushed 1:41df217a:::smithi13140796-39:head to all replicas 2019-12-07T19:34:04.700+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] finish_recovery_op 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.700+0000 7fe2caa6c700 10 osd.0 27 finish_recovery_op pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] 1:41df217a:::smithi13140796-39:head dequeue=0 (1/10 rops) 2019-12-07T19:34:04.700+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] queue_recovery -- queuing 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) 2019-12-07T19:34:04.700+0000 7fe2caa6c700 10 osd.0 27 _maybe_queue_recovery starting 1, recovery_ops_reserved 0 -> 1 2019-12-07T19:34:04.700+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.700+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) queued 2019-12-07T19:34:04.700+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] release_backoffs [1:41df217a:::smithi13140796-39:head,1:41df217a:::smithi13140796-39:head) 2019-12-07T19:34:04.700+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] finish_degraded_object 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.700+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] continue_recovery_op: WRITING return RecoveryOp(hoid=1:41df217a:::smithi13140796-39:head v=21'5 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41df217a:::smithi13140796-39:head@21'5, size: 638976, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:638976, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=2 state=COMPLETE waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:04.700+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e818051e0 finish 2019-12-07T19:34:04.700+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.700+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) pg 0x558e81075000 2019-12-07T19:34:04.700+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.700+0000 7fe2c6a64700 10 osd.0 27 do_recovery starting 1 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] 2019-12-07T19:34:04.700+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] recover_replicas(1) 2019-12-07T19:34:04.700+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] peer osd.5(3) missing 2 objects. 2019-12-07T19:34:04.700+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] peer osd.5(3) missing {1:41fe8a11:::smithi13140796-45:head=21'6 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] recover_replicas: recover_object_replicas(1:41fe8a11:::smithi13140796-45:head) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] prep_object_replica_pushes: on 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_object_context: found obc in cache: 0x558e8115b440 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_object_context: 0x558e8115b440 1:41fe8a11:::smithi13140796-45:head rwstate(none n=0 w=0) oi: 1:41fe8a11:::smithi13140796-45:head(21'6 client.4540.0:134 dirty|data_digest s 753664 uv 6 dd 1117d0c7 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e83dcc900 snapset: 0=[]:{} 2019-12-07T19:34:04.701+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] recovery got recovery read lock on 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] start_recovery_op 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 27 start_recovery_op pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] 1:41fe8a11:::smithi13140796-45:head (0/10 rops) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] checking 0(0) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] checking 1(2) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] checking 3(5) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] checking 4(1) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] checking 5(3) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] checking 7(4) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] recover_object: built op RecoveryOp(hoid=1:41fe8a11:::smithi13140796-45:head v=21'6 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=5 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] peer osd.7(4) missing 2 objects. 2019-12-07T19:34:04.701+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] peer osd.7(4) missing {1:41fe8a11:::smithi13140796-45:head=21'6 flags = none clean_offsets: [], clean_omap: 0, new_object: 1,1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] run_recovery_op: starting RecoveryOp(hoid=1:41fe8a11:::smithi13140796-45:head v=21'6 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:41fe8a11:::smithi13140796-45:head v=21'6 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=4 state=IDLE waiting_on_pushes= extent_requested=0,0) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_hash_info: Getting attr on 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_hash_info: not in cache 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_hash_info: found on disk, size 188416 2019-12-07T19:34:04.701+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# hinfo_key = 0 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_all_avail_shards: checking acting 0(0) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_all_avail_shards: checking acting 1(2) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_all_avail_shards: checking acting 3(5) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_all_avail_shards: checking acting 4(1) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_all_avail_shards: checking missing_loc 0(0) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_all_avail_shards: checking missing_loc 1(2) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_all_avail_shards: checking missing_loc 3(5) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_all_avail_shards: checking missing_loc 4(1) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_all_avail_shards: checking missing_loc 5(4) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] continue_recovery_op: IDLE return RecoveryOp(hoid=1:41fe8a11:::smithi13140796-45:head v=21'6 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=4 state=READING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] start_read_op: starting ReadOp(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] do_read_op: starting read ReadOp(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={}, source_to_obj={}, in_progress=) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e84242780 con 0x558e802a5400 2019-12-07T19:34:04.701+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpRead(1.2s2 27/26 ECSubRead(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e84243900 con 0x558e818d7c00 2019-12-07T19:34:04.701+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 ==== 0+0+0 (unknown 0 0 0) 0x558e84242780 con 0x558e802a5400 2019-12-07T19:34:04.701+0000 7fe2d427f700 15 osd.0 27 enqueue_op 0x558e81943080 prio 3 cost 0 latency 0.000018 epoch 27 MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.701+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpRead(1.2s1 27/26 ECSubRead(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e8417c000 con 0x558e818fe400 2019-12-07T19:34:04.701+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpRead(1.2s4 27/26 ECSubRead(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 -- 0x558e84243400 con 0x558e818d6800 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] do_read_op: started ReadOp(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, []))}, priority=3, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head,5(4)=1:41fe8a11:::smithi13140796-45:head}, in_progress=0(0),1(2),4(1),5(4)) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] started 1 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 27 do_recovery started 1/1 on pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 27 release_reserved_pushes(1), recovery_ops_reserved 1 -> 0 2019-12-07T19:34:04.701+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.701+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81943080 prio 3 cost 0 latency 0.000113 MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.2s0 27/26 ECSubRead(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=0,262144,0}, subchunks={1:41fe8a11:::smithi13140796-45:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.701+0000 7fe2caa6c700 25 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.701+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 0x0~40000 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~40000 size 0x2e000 (188416) 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840308c0 blob([0x216000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84030930 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84030850 blob([0x218000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840307e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031260 blob([0x21a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031110 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840312d0 blob([0x21c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031340 blob([0x21e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840311f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840313b0 blob([0x220000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031490 blob([0x222000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031500 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031570 blob([0x224000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840315e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031650 blob([0x226000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840316c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031730 blob([0x228000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840317a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031810 blob([0x22a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff29a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ff2c40 blob([0x22c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ff25b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031880 blob([0x22e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840318f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031960 blob([0x230000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840319d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031a40 blob([0x232000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031ab0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031b20 blob([0x234000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031b90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031c00 blob([0x236000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031c70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031ce0 blob([0x238000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031d50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031dc0 blob([0x23a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031e30 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031ea0 blob([0x23c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84031f10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84031f80 blob([0x23e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede000 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ede070 blob([0x240000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede0e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ede150 blob([0x242000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ede1c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:04.701+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.2s0_head 0#1:41fe8a11:::smithi13140796-45:head# 0x0~40000 = 188416 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read read request=262144 r=188416 len=188416 2019-12-07T19:34:04.701+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] get_hash_info: Getting attr on 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read: Checking hash of 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.701+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] -- MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 -- 0x558e84243b80 con 0x558e802a5400 2019-12-07T19:34:04.701+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81943080 finish 2019-12-07T19:34:04.701+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.701+0000 7fe2d427f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.0 v2:172.21.15.131:6822/37079 0 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 ==== 0+0+0 (unknown 0 0 0) 0x558e84243b80 con 0x558e802a5400 2019-12-07T19:34:04.701+0000 7fe2d427f700 15 osd.0 27 enqueue_op 0x558e819438c0 prio 127 cost 0 latency 0.000015 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 2019-12-07T19:34:04.701+0000 7fe2d427f700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.701+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 157 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 ==== 188581+0+0 (unknown 0 0 0) 0x558e84243900 con 0x558e818d7c00 2019-12-07T19:34:04.701+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edd1e0 prio 127 cost 0 latency 0.000015 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 2019-12-07T19:34:04.701+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.701+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.701+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.701+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e819438c0 prio 127 cost 0 latency 0.000224 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=22, attrs_read=0) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [0(0),188416]))}, priority=3, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head,5(4)=1:41fe8a11:::smithi13140796-45:head}, in_progress=1(2),4(1),5(4)) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e819438c0 finish 2019-12-07T19:34:04.701+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.701+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.701+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.701+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edd1e0 prio 127 cost 0 latency 0.000176 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=22, attrs_read=0) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [0(0),188416, 1(2),188416]))}, priority=3, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head,5(4)=1:41fe8a11:::smithi13140796-45:head}, in_progress=4(1),5(4)) 2019-12-07T19:34:04.701+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edd1e0 finish 2019-12-07T19:34:04.701+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.702+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 84 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 ==== 188581+0+0 (unknown 0 0 0) 0x558e8417c000 con 0x558e818fe400 2019-12-07T19:34:04.702+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e83edc000 prio 127 cost 0 latency 0.000013 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 2019-12-07T19:34:04.702+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.702+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.702+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.702+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.702+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.702+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edc000 prio 127 cost 0 latency 0.000036 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] 2019-12-07T19:34:04.702+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 2019-12-07T19:34:04.702+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=22, attrs_read=0) 2019-12-07T19:34:04.702+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read_reply readop not complete: ReadOp(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [0(0),188416, 1(2),188416, 4(1),188416]))}, priority=3, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head,5(4)=1:41fe8a11:::smithi13140796-45:head}, in_progress=5(4)) 2019-12-07T19:34:04.702+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edc000 finish 2019-12-07T19:34:04.702+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.705+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 40 ==== MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 ==== 188581+0+0 (unknown 0 0 0) 0x558e84243400 con 0x558e818d6800 2019-12-07T19:34:04.705+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e81805ce0 prio 127 cost 0 latency 0.000016 epoch 27 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 2019-12-07T19:34:04.705+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.705+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.705+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.705+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.705+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2) prio 127 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.705+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e81805ce0 prio 127 cost 0 latency 0.000037 MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] 2019-12-07T19:34:04.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] _handle_message: MOSDECSubOpReadReply(1.2s0 27/26 ECSubReadReply(tid=22, attrs_read=0)) v2 2019-12-07T19:34:04.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read_reply: reply ECSubReadReply(tid=22, attrs_read=0) 2019-12-07T19:34:04.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read_reply have shard=0 2019-12-07T19:34:04.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read_reply have shard=2 2019-12-07T19:34:04.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read_reply have shard=1 2019-12-07T19:34:04.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read_reply have shard=4 2019-12-07T19:34:04.705+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_sub_read_reply Complete: ReadOp(tid=22, to_read={1:41fe8a11:::smithi13140796-45:head=read_request_t(to_read=[0,1048576,0], need={0(0)=[0,8],1(2)=[0,8],4(1)=[0,8],5(4)=[0,8]}, want_attrs=0)}, complete={1:41fe8a11:::smithi13140796-45:head=read_result_t(r=0, errors={}, noattrs, returned=(0, 1048576, [0(0),188416, 1(2),188416, 4(1),188416, 5(4),188416]))}, priority=3, obj_to_source={1:41fe8a11:::smithi13140796-45:head=0(0),1(2),4(1),5(4)}, source_to_obj={0(0)=1:41fe8a11:::smithi13140796-45:head,1(2)=1:41fe8a11:::smithi13140796-45:head,4(1)=1:41fe8a11:::smithi13140796-45:head,5(4)=1:41fe8a11:::smithi13140796-45:head}, in_progress=) 2019-12-07T19:34:04.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_recovery_read_complete: returned 1:41fe8a11:::smithi13140796-45:head (0, 1048576, [0(0),188416, 1(2),188416, 4(1),188416, 5(4),188416]) 2019-12-07T19:34:04.705+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] handle_recovery_read_complete: [0,188416, 1,188416, 2,188416, 4,188416] 2019-12-07T19:34:04.705+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 375319 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.706+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:41fe8a11:::smithi13140796-45:head v=21'6 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=READING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:04.706+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] continue_recovery_op: before_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), after_progress=ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), pop.data.length()=188416, size=753664 2019-12-07T19:34:04.706+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] continue_recovery_op: before_progress=ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), after_progress=ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), pop.data.length()=188416, size=753664 2019-12-07T19:34:04.707+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] continue_recovery_op: READING return RecoveryOp(hoid=1:41fe8a11:::smithi13140796-45:head v=21'6 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=WRITING waiting_on_pushes=5(3),7(4) extent_requested=0,1048576) 2019-12-07T19:34:04.707+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDPGPush(1.2s3 27/26 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 -- 0x558e84028800 con 0x558e818d6800 2019-12-07T19:34:04.707+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDPGPush(1.2s4 27/26 [PushOp(1:41fe8a11:::smithi13140796-45:head, version: 21'6, data_included: [0~188416], data_size: 188416, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 -- 0x558e83ca8a00 con 0x558e818fe000 2019-12-07T19:34:04.707+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e81805ce0 finish 2019-12-07T19:34:04.707+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.708+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 23 ==== MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3 ==== 111+0+0 (unknown 0 0 0) 0x558e83ca8a00 con 0x558e818fe000 2019-12-07T19:34:04.708+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e81943600 prio 127 cost 1049576 latency 0.000019 epoch 27 MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3 2019-12-07T19:34:04.708+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3) prio 127 cost 1049576 e27) 2019-12-07T19:34:04.708+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.708+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3) prio 127 cost 1049576 e27) queued 2019-12-07T19:34:04.708+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.708+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3) prio 127 cost 1049576 e27) pg 0x558e81075000 2019-12-07T19:34:04.708+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81943600 prio 127 cost 1049576 latency 0.000052 MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] 2019-12-07T19:34:04.708+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] _handle_message: MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3 2019-12-07T19:34:04.708+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:41fe8a11:::smithi13140796-45:head v=21'6 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=WRITING waiting_on_pushes=5(3) extent_requested=0,1048576) 2019-12-07T19:34:04.708+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81943600 finish 2019-12-07T19:34:04.708+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.708+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 41 ==== MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3 ==== 111+0+0 (unknown 0 0 0) 0x558e84029200 con 0x558e818d6800 2019-12-07T19:34:04.708+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e81942c60 prio 127 cost 1049576 latency 0.000012 epoch 27 MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3 2019-12-07T19:34:04.708+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3) prio 127 cost 1049576 e27) 2019-12-07T19:34:04.708+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.708+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3) prio 127 cost 1049576 e27) queued 2019-12-07T19:34:04.708+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.708+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3) prio 127 cost 1049576 e27) pg 0x558e81075000 2019-12-07T19:34:04.708+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e81942c60 prio 127 cost 1049576 latency 0.000047 MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] 2019-12-07T19:34:04.709+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] _handle_message: MOSDPGPushReply(1.2s0 27/26 [PushReplyOp(1:41fe8a11:::smithi13140796-45:head)]) v3 2019-12-07T19:34:04.709+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] continue_recovery_op: continuing RecoveryOp(hoid=1:41fe8a11:::smithi13140796-45:head v=21'6 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=3 state=WRITING waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:04.709+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] continue_recovery_op: on_peer_recover on 5(3), obj 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 2 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 2 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1)=2,0(0),1(2),3(5),4(1),5(4)=1} 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 0 ml 1 missing shards 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 1 ml 1 missing shards 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 2 ml 1 missing shards 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 3 ml 1 missing shards 1 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 4 ml 1 missing shards 1 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 5 ml 1 missing shards 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats missing based degraded 1 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats missing based misplaced 1 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats degraded 1 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats misplaced 1 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.709+0000 7fe2c6a64700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(0+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] publish_stats_to_osd 27:59 2019-12-07T19:34:04.709+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] continue_recovery_op: on_peer_recover on 7(4), obj 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 1 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 2 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1)=2,0(0),1(2),3(5),4(1),5(3),5(4)=1} 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 0 ml 1 missing shards 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 1 ml 1 missing shards 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 2 ml 1 missing shards 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 3 ml 1 missing shards 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 4 ml 1 missing shards 1 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats shard 5 ml 1 missing shards 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats missing based degraded 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats missing based misplaced 1 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] update_calc_stats misplaced 1 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.709+0000 7fe2c6a64700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(1+0)=1},1={(1+0)=1},2={(1+0)=1},3={(1+0)=1},4={(0+1)=1},5={(1+0)=1}} ps=[2~1]] publish_stats_to_osd 27:60 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 1 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 1 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1)=3} 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing based degraded 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing based misplaced 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.709+0000 7fe2c6a64700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] publish_stats_to_osd 27:61 2019-12-07T19:34:04.709+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] pushed 1:41fe8a11:::smithi13140796-45:head to all replicas 2019-12-07T19:34:04.709+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] finish_recovery_op 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.709+0000 7fe2c6a64700 10 osd.0 27 finish_recovery_op pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 1:41fe8a11:::smithi13140796-45:head dequeue=0 (1/10 rops) 2019-12-07T19:34:04.709+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] queue_recovery -- queuing 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) 2019-12-07T19:34:04.709+0000 7fe2c6a64700 10 osd.0 27 _maybe_queue_recovery starting 1, recovery_ops_reserved 0 -> 1 2019-12-07T19:34:04.709+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.709+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] release_backoffs [1:41fe8a11:::smithi13140796-45:head,1:41fe8a11:::smithi13140796-45:head) 2019-12-07T19:34:04.709+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) queued 2019-12-07T19:34:04.709+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] finish_degraded_object 1:41fe8a11:::smithi13140796-45:head 2019-12-07T19:34:04.709+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] continue_recovery_op: WRITING return RecoveryOp(hoid=1:41fe8a11:::smithi13140796-45:head v=21'6 missing_on=5(3),7(4) missing_on_shards=3,4 recovery_info=ObjectRecoveryInfo(1:41fe8a11:::smithi13140796-45:head@21'6, size: 753664, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1) recovery_progress=ObjectRecoveryProgress(!first, data_recovered_to:753664, data_complete:true, omap_recovered_to:, omap_complete:true, error:false) obc refcount=2 state=COMPLETE waiting_on_pushes= extent_requested=0,1048576) 2019-12-07T19:34:04.709+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e81942c60 finish 2019-12-07T19:34:04.709+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.709+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.709+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) pg 0x558e81075000 2019-12-07T19:34:04.709+0000 7fe2caa6c700 10 osd.0 27 do_recovery starting 1 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.709+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] recover_replicas(1) 2019-12-07T19:34:04.709+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] peer osd.5(3) missing 1 objects. 2019-12-07T19:34:04.709+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] peer osd.5(3) missing {1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.709+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] recover_replicas: 1:4c9e444c:::smithi13140796-48:head is a delete, removing 2019-12-07T19:34:04.709+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] prep_object_replica_deletes: on 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:04.709+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] get_object_context: obc NOT found in cache: 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:04.709+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# _ 2019-12-07T19:34:04.709+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# _ = -2 2019-12-07T19:34:04.709+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] get_object_context: no obc for soid 1:4c9e444c:::smithi13140796-48:head and !can_create 2019-12-07T19:34:04.709+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] start_recovery_op 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:04.709+0000 7fe2caa6c700 10 osd.0 27 start_recovery_op pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 1:4c9e444c:::smithi13140796-48:head (0/10 rops) 2019-12-07T19:34:04.709+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] recover_delete_object will remove 1:4c9e444c:::smithi13140796-48:head 23'10 from 5(3) 2019-12-07T19:34:04.709+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] recover_delete_object will remove 1:4c9e444c:::smithi13140796-48:head 23'10 from 7(4) 2019-12-07T19:34:04.709+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] peer osd.7(4) missing 1 objects. 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] peer osd.7(4) missing {1:4c9e444c:::smithi13140796-48:head=23'10 flags = delete clean_offsets: [], clean_omap: 0, new_object: 1} 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] send_recovery_deletes: sending recovery delete << 1:4c9e444c:::smithi13140796-48:head 23'10 to osd.5(3) 2019-12-07T19:34:04.710+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDPGRecoveryDelete(1.2s3 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2 -- 0x558e83ca8a00 con 0x558e818d6800 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] send_recovery_deletes: sending recovery delete << 1:4c9e444c:::smithi13140796-48:head 23'10 to osd.7(4) 2019-12-07T19:34:04.710+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDPGRecoveryDelete(1.2s4 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2 -- 0x558e81955600 con 0x558e818fe000 2019-12-07T19:34:04.710+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] started 1 2019-12-07T19:34:04.710+0000 7fe2caa6c700 10 osd.0 27 do_recovery started 1/1 on pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.710+0000 7fe2caa6c700 10 osd.0 27 release_reserved_pushes(1), recovery_ops_reserved 1 -> 0 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.710+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 24 ==== pg_trim(1.2s0 23'10 e27/27) v2 ==== 34+0+0 (unknown 0 0 0) 0x558e819b14a0 con 0x558e818fe000 2019-12-07T19:34:04.710+0000 7fe2e753b700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 MTrim epoch 27 from 7 shard 0 trim_to 23'10 2019-12-07T19:34:04.710+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MTrim epoch 27 from 7 shard 0 trim_to 23'10) prio 255 cost 10 e27) 2019-12-07T19:34:04.710+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.710+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 25 ==== MOSDPGRecoveryDeleteReply(1.2s0 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2 ==== 109+0+0 (unknown 0 0 0) 0x558e83ca8a00 con 0x558e818fe000 2019-12-07T19:34:04.710+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MTrim epoch 27 from 7 shard 0 trim_to 23'10) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.710+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.710+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e81943b80 prio 3 cost 0 latency 0.000008 epoch 27 MOSDPGRecoveryDeleteReply(1.2s0 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2 2019-12-07T19:34:04.710+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MTrim epoch 27 from 7 shard 0 trim_to 23'10) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.710+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDeleteReply(1.2s0 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2) prio 3 cost 0 e27) 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDeleteReply(1.2s0 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.710+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MTrim epoch 27 from 7 shard 0 trim_to 23'10 2019-12-07T19:34:04.710+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] state: replica osd.7 lcod 23'10 2019-12-07T19:34:04.710+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDeleteReply(1.2s0 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2) prio 3 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.710+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.710+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81943b80 prio 3 cost 0 latency 0.000064 MOSDPGRecoveryDeleteReply(1.2s0 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] handle_recovery_delete_reply 0x558e81943b80 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 1 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 1 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1)=3} 2019-12-07T19:34:04.710+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 42 ==== pg_trim(1.2s0 23'10 e27/27) v2 ==== 34+0+0 (unknown 0 0 0) 0x558e819b14a0 con 0x558e818d6800 2019-12-07T19:34:04.710+0000 7fe2e753b700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 MTrim epoch 27 from 5 shard 0 trim_to 23'10 2019-12-07T19:34:04.710+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MTrim epoch 27 from 5 shard 0 trim_to 23'10) prio 255 cost 10 e27) 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing based degraded 0 2019-12-07T19:34:04.710+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing based misplaced 0 2019-12-07T19:34:04.710+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MTrim epoch 27 from 5 shard 0 trim_to 23'10) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.710+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] publish_stats_to_osd 27:62 2019-12-07T19:34:04.710+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 43 ==== MOSDPGRecoveryDeleteReply(1.2s0 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2 ==== 109+0+0 (unknown 0 0 0) 0x558e81955600 con 0x558e818d6800 2019-12-07T19:34:04.710+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e81942f20 prio 3 cost 0 latency 0.000011 epoch 27 MOSDPGRecoveryDeleteReply(1.2s0 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2 2019-12-07T19:34:04.710+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDeleteReply(1.2s0 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2) prio 3 cost 0 e27) 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] handle_recovery_delete_reply 1:4c9e444c:::smithi13140796-48:head still missing on at least 5(3) 2019-12-07T19:34:04.710+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81943b80 finish 2019-12-07T19:34:04.710+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.710+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 MTrim epoch 27 from 5 shard 0 trim_to 23'10) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDeleteReply(1.2s0 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.710+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 MTrim epoch 27 from 5 shard 0 trim_to 23'10 2019-12-07T19:34:04.710+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] state: replica osd.5 lcod 23'10 2019-12-07T19:34:04.710+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.710+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.710+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDPGRecoveryDeleteReply(1.2s0 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2) prio 3 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.710+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81942f20 prio 3 cost 0 latency 0.000070 MOSDPGRecoveryDeleteReply(1.2s0 e27,26 1:4c9e444c:::smithi13140796-48:head,23'10) v2 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] handle_recovery_delete_reply 0x558e81942f20 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 1 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1)=2,7(4)=1} 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats shard 0 ml 1 missing shards 1 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats shard 1 ml 1 missing shards 1 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats shard 2 ml 1 missing shards 1 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats shard 3 ml 1 missing shards 1 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats shard 4 ml 1 missing shards 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats shard 5 ml 1 missing shards 1 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats missing based degraded 5 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats missing based misplaced 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats degraded 5 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.711+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(0+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] publish_stats_to_osd 27:63 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={(0+0)=1},1={(0+0)=1},2={(0+0)=1},3={(1+0)=1},4={(1+0)=1},5={(0+0)=1}} ps=[2~1]] handle_recovery_delete_reply completed recovery, local_missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1)=3} 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing based degraded 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing based misplaced 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+degraded+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.711+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] publish_stats_to_osd 27:64 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] pushed 1:4c9e444c:::smithi13140796-48:head to all replicas 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped rops=1 mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] finish_recovery_op 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 27 finish_recovery_op pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 1:4c9e444c:::smithi13140796-48:head dequeue=0 (1/10 rops) 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] queue_recovery -- queuing 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 27 _maybe_queue_recovery starting 1, recovery_ops_reserved 0 -> 1 2019-12-07T19:34:04.711+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.711+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) queued 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] release_backoffs [1:4c9e444c:::smithi13140796-48:head,1:4c9e444c:::smithi13140796-48:head) 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] finish_degraded_object 1:4c9e444c:::smithi13140796-48:head 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e81942f20 finish 2019-12-07T19:34:04.711+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.711+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGRecovery(pgid=1.2s0epoch_queued=27reserved_pushes=1) prio 5 cost 20971520 e27 reserved_pushes 1) pg 0x558e81075000 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.711+0000 7fe2c6a64700 10 osd.0 27 do_recovery starting 1 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.711+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] recover_replicas(1) 2019-12-07T19:34:04.711+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] recover_primary recovering 0 in pg, missing missing(0 may_include_deletes = 1) 2019-12-07T19:34:04.711+0000 7fe2c6a64700 25 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] recover_primary {} 2019-12-07T19:34:04.711+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] started 0 2019-12-07T19:34:04.711+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] start_recovery_ops needs_recovery: {} 2019-12-07T19:34:04.711+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] start_recovery_ops missing_loc: {} 2019-12-07T19:34:04.711+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+recovering+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] needs_recovery is recovered 2019-12-07T19:34:04.711+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] needs_backfill does not need backfill 2019-12-07T19:34:04.711+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] recovery done, no backfill 2019-12-07T19:34:04.711+0000 7fe2c6a64700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 27 epoch_requested: 27 AllReplicasRecovered 2019-12-07T19:34:04.711+0000 7fe2c6a64700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 AllReplicasRecovered) prio 255 cost 10 e27) 2019-12-07T19:34:04.711+0000 7fe2c6a64700 10 osd.0 27 do_recovery started 0/1 on pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.711+0000 7fe2c6a64700 10 osd.0 27 release_reserved_pushes(1), recovery_ops_reserved 1 -> 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 AllReplicasRecovered) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 AllReplicasRecovered) prio 255 cost 10 e27) pg 0x558e81075000 2019-12-07T19:34:04.711+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 AllReplicasRecovered 2019-12-07T19:34:04.711+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.2s2 RELEASE e27/27) v3 -- 0x558e819c4780 con 0x558e818d7c00 2019-12-07T19:34:04.711+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MRecoveryReserve(1.2s5 RELEASE e27/27) v3 -- 0x558e84199860 con 0x558e818d7000 2019-12-07T19:34:04.711+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MRecoveryReserve(1.2s1 RELEASE e27/27) v3 -- 0x558e841983c0 con 0x558e818fe400 2019-12-07T19:34:04.711+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MRecoveryReserve(1.2s3 RELEASE e27/27) v3 -- 0x558e84198000 con 0x558e818d6800 2019-12-07T19:34:04.711+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MRecoveryReserve(1.2s4 RELEASE e27/27) v3 -- 0x558e819b05a0 con 0x558e818fe000 2019-12-07T19:34:04.711+0000 7fe2caa6c700 5 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] exit Started/Primary/Active/Recovering 2019-12-07T19:34:04.712355+0000 3 0.000216 2019-12-07T19:34:04.711+0000 7fe2caa6c700 5 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] enter Started/Primary/Active/Recovered 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] needs_recovery is recovered 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1)=3} 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.711+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.711+0000 7fe2caa6c700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] publish_stats_to_osd 27:65 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] choose_acting all_info osd.0(0) 1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] choose_acting all_info osd.1(2) 1.2s2( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] choose_acting all_info osd.3(5) 1.2s5( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] choose_acting all_info osd.4(1) 1.2s1( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] choose_acting all_info osd.5(3) 1.2s3( v 23'10 lc 0'0 (0'0,23'10] local-lis/les=26/27 n=0 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=0) 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] choose_acting all_info osd.5(4) 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] choose_acting all_info osd.7(4) 1.2s4( v 23'10 lc 0'0 (0'0,23'10] local-lis/les=26/27 n=0 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=0) 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] find_best_info skipping osd.5(3) because it has missing while best is complete 2019-12-07T19:34:04.711+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] find_best_info skipping osd.7(4) because it has missing while best is complete 2019-12-07T19:34:04.712+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] For position 0: selecting up[i]: 0(0) For position 1: selecting up[i]: 4(1) For position 2: selecting up[i]: 1(2) For position 3: selecting up[i]: 5(3) For position 4: selecting up[i]: 7(4) For position 5: selecting up[i]: 3(5) 2019-12-07T19:34:04.712+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] choose_async_recovery_ec candidates by cost are: 2019-12-07T19:34:04.712+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] choose_async_recovery_ec result want=[0,4,1,5,7,3] async_recovery= 2019-12-07T19:34:04.712+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] choose_acting want [0,4,1,5,7,3] != acting [0,4,1,2147483647,2147483647,3], requesting pg_temp change 2019-12-07T19:34:04.712+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.712+0000 7fe2caa6c700 10 osd.0 27 send_pg_temp {1.2=[]} 2019-12-07T19:34:04.712+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- osd_pgtemp(e27 {1.2=[]} v27) v2 -- 0x558e8419cfc0 con 0x558e80f39000 2019-12-07T19:34:04.712+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.721+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 88 ==== MRecoveryReserve(1.fs1 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e84199860 con 0x558e818d7000 2019-12-07T19:34:04.721+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.fs1 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.721+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.721+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.721+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.721+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.721+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.721+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.721+0000 7fe2c425f700 5 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.721915+0000 4 0.000109 2019-12-07T19:34:04.721+0000 7fe2c425f700 5 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.721+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:04.721+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.725+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 18 ==== osd_op(client.4540.0:231 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e27) v8 ==== 234+0+0 (crc 0 0 0) 0x558e83ca6b00 con 0x558e83d68000 2019-12-07T19:34:04.725+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e83edc420 prio 63 cost 0 latency 0.000024 epoch 27 osd_op(client.4540.0:231 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e27) v8 2019-12-07T19:34:04.725+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:231 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e27) v8) prio 63 cost 0 e27) 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:231 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e27) v8) prio 63 cost 0 e27) queued 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:231 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e27) v8) prio 63 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.725+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edc420 prio 63 cost 0 latency 0.000063 osd_op(client.4540.0:231 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e27) v8 pg pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.725+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] _handle_message: osd_op(client.4540.0:231 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e27) v8 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] do_op: op osd_op(client.4540.0:231 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 4=[4] ondisk+write+peerstat_old+known_if_redirected e27) v8 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:04.725+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] do_op osd_op(client.4540.0:231 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 4=[4] ondisk+write+peerstat_old+known_if_redirected e27) v8 may_write -> write-ordered flags ondisk+write+peerstat_old+known_if_redirected 2019-12-07T19:34:04.725+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] get_object_context: found obc in cache: 0x558e8405d200 2019-12-07T19:34:04.725+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] get_object_context: 0x558e8405d200 1:41df217a:::smithi13140796-39:head rwstate(none n=0 w=0) oi: 1:41df217a:::smithi13140796-39:head(21'5 client.4540.0:116 dirty|data_digest s 638976 uv 5 dd b038798b alloc_hint [0 0 0]) exists: 1 ssc: 0x558e83dccb40 snapset: 0=[]:{} 2019-12-07T19:34:04.725+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] find_object_context 1:41df217a:::smithi13140796-39:head @head oi=1:41df217a:::smithi13140796-39:head(21'5 client.4540.0:116 dirty|data_digest s 638976 uv 5 dd b038798b alloc_hint [0 0 0]) 2019-12-07T19:34:04.725+0000 7fe2c6a64700 25 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] do_op oi 1:41df217a:::smithi13140796-39:head(21'5 client.4540.0:116 dirty|data_digest s 638976 uv 5 dd b038798b alloc_hint [0 0 0]) 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] do_op obc obc(1:41df217a:::smithi13140796-39:head rwstate(write n=1 w=0)) 2019-12-07T19:34:04.725+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] execute_ctx 0x558e8122b100 2019-12-07T19:34:04.725+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] execute_ctx 1:41df217a:::smithi13140796-39:head [delete] ov 21'5 av 27'11 snapc 4=[4] snapset 0=[]:{} 2019-12-07T19:34:04.725+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] do_osd_op 1:41df217a:::smithi13140796-39:head [delete] 2019-12-07T19:34:04.725+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] do_osd_op delete 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] _delete_oid has or will have clones; will whiteout 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] _delete_oid 1:41df217a:::smithi13140796-39:head whiteout=1 no_whiteout=0 try_no_whiteout=0 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] _delete_oid setting whiteout on 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] make_writeable 1:41df217a:::smithi13140796-39:head snapset=0=[]:{} snapc=4=[4] 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] got greedy write on clone_obc obc(1:41df217a:::smithi13140796-39:4 rwstate(write n=1 w=0)) 2019-12-07T19:34:04.725+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] cloning v 21'5 to 1:41df217a:::smithi13140796-39:4 v 27'11 snaps=[4] snapset=0=[]:{4=[4]} 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] make_writeable 1:41df217a:::smithi13140796-39:head done, snapset=4=[]:{4=[4]} 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] finish_ctx 1:41df217a:::smithi13140796-39:head 0x558e8122b100 op modify 2019-12-07T19:34:04.725+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] set mtime to 2019-12-07T19:34:04.726120+0000 2019-12-07T19:34:04.725+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] final snapset 4=[]:{4=[4]} in 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] finish_ctx object 1:41df217a:::smithi13140796-39:head marks clean_regions clean_offsets: [638976~18446744073708912639], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.725+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] execute_ctx alloc reply 0x558e84243400 result 0 2019-12-07T19:34:04.725+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] calc_trim_to_aggressive limit = 23'10 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] op order client.4540 tid 231 (first) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] new_repop rep_tid 23 on osd_op(client.4540.0:231 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 4=[4] ondisk+write+peerstat_old+known_if_redirected e27) v8 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] new_repop: repgather(0x558e81123d40 0'0 rep_tid=23 committed?=0 r=0) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 7 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] issue_repop rep_tid 23 o 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] submit_transaction: op Op(1:41df217a:::smithi13140796-39:head v=27'12 tt=0'0 tid=23 reqid=client.4540.0:231 client_op=osd_op(client.4540.0:231 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 4=[4] ondisk+write+peerstat_old+known_if_redirected e27) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] get_hash_info: Getting attr on 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] get_hash_info: not in cache 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:41df217a:::smithi13140796-39:4# key 0x80800000000000000141df21'z!smithi13140796-39!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] get_hash_info: Getting attr on 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] get_hash_info: not in cache 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] get_hash_info: found on disk, size 159744 2019-12-07T19:34:04.726+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key = 0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] operator(): 1:41df217a:::smithi13140796-39:4 projected size 638976 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] get_hash_info: Getting attr on 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] operator(): delete, setting projected size to 0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] operator(): 1:41df217a:::smithi13140796-39:head projected size 0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] start_rmw: Op(1:41df217a:::smithi13140796-39:head v=27'12 tt=0'0 tid=23 reqid=client.4540.0:231 client_op=osd_op(client.4540.0:231 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 4=[4] ondisk+write+peerstat_old+known_if_redirected e27) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:4=[],1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_state_to_reads: invalidating cache after this op 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_state_to_reads: Op(1:41df217a:::smithi13140796-39:head v=27'12 tt=0'0 tid=23 reqid=client.4540.0:231 client_op=osd_op(client.4540.0:231 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 4=[4] ondisk+write+peerstat_old+known_if_redirected e27) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:4=[],1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_reads_to_commit: starting commit on Op(1:41df217a:::smithi13140796-39:head v=27'12 tt=0'0 tid=23 reqid=client.4540.0:231 client_op=osd_op(client.4540.0:231 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 4=[4] ondisk+write+peerstat_old+known_if_redirected e27) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:4=[],1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] generate_transactions: 1:41df217a:::smithi13140796-39:4, current size is 0 buffers are {} 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] operator(): new_size start 638976 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] operator(): to_overwrite: {} 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] operator(): to_append: {} 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] operator(): 1:41df217a:::smithi13140796-39:4 resetting hinfo to logical size 638976 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] generate_transactions: 1:41df217a:::smithi13140796-39:head, current size is 638976 buffers are {} 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] operator(): new_size start 0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] operator(): to_overwrite: {} 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] operator(): to_append: {} 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] operator(): 1:41df217a:::smithi13140796-39:head resetting hinfo to logical size 0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_reads_to_commit: written: {1:41df217a:::smithi13140796-39:4={},1:41df217a:::smithi13140796-39:head={}} 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_reads_to_commit: op: Op(1:41df217a:::smithi13140796-39:head v=27'12 tt=0'0 tid=23 reqid=client.4540.0:231 client_op=osd_op(client.4540.0:231 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 4=[4] ondisk+write+peerstat_old+known_if_redirected e27) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:4=[],1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_reads_to_commit: written_set: {1:41df217a:::smithi13140796-39:4=[],1:41df217a:::smithi13140796-39:head=[]} 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_reads_to_commit: 1:41df217a:::smithi13140796-39:4,{} 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_reads_to_commit: 1:41df217a:::smithi13140796-39:head,{} 2019-12-07T19:34:04.726+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 27/26 ECSubWrite(tid=23, reqid=client.4540.0:231, at_version=27'12, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e84042000 con 0x558e818d7c00 2019-12-07T19:34:04.726+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 27/26 ECSubWrite(tid=23, reqid=client.4540.0:231, at_version=27'12, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e84042800 con 0x558e818d7000 2019-12-07T19:34:04.726+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 27/26 ECSubWrite(tid=23, reqid=client.4540.0:231, at_version=27'12, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e811b9800 con 0x558e818fe400 2019-12-07T19:34:04.726+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s3 27/26 ECSubWrite(tid=23, reqid=client.4540.0:231, at_version=27'12, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e819db000 con 0x558e818d6800 2019-12-07T19:34:04.726+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=23, reqid=client.4540.0:231, at_version=27'12, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e819d8800 con 0x558e818fe000 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 snap_mapper.add_oid 1:41df217a:::smithi13140796-39:4 4 2019-12-07T19:34:04.726+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 snap_mapper.get_snaps 1:41df217a:::smithi13140796-39:4 got.empty() 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 snap_mapper.set_snaps 1:41df217a:::smithi13140796-39:4 4 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 snap_mapper.set_snaps set OBJ_.0_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 snap_mapper.add_oid set SNA_1_0000000000000004_.0_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 earliest_dup_version = 0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 trim 27'11 (21'5) clone 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:33:58.731948+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 trim 27'12 (21'5) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:231 2019-12-07T19:34:04.726120+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 23'10 (0'0,23'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] append_log log((0'0,23'10], crt=23'10) [27'11 (21'5) clone 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:33:58.731948+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,27'12 (21'5) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:231 2019-12-07T19:34:04.726120+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 0, new_object: 0] 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'11 (0'0,27'11] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 lua=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] add_log_entry 27'11 (21'5) clone 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:33:58.731948+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 lua=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] add_log_entry 27'12 (21'5) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:231 2019-12-07T19:34:04.726120+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 lua=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] append_log approx pg log length = 12 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 lua=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e8417a300 seq 27 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:41df217a:::smithi13140796-39:4# key 0x80800000000000000141df21'z!smithi13140796-39!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _clone 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# -> 0#1:41df217a:::smithi13140796-39:4# 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1128 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.onode(0x558e83cb3900).flush flush done 2019-12-07T19:34:04.726+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# 0x0 2019-12-07T19:34:04.726+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_clone_range 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# -> 0#1:41df217a:::smithi13140796-39:4# 0x0~27000 -> 0x0~27000 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x0~2000: 0x0~2000 Blob(0x558e83dd2ee0 blob([0x1c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e70 sbid 0x0)) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10260 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd2ee0 blob([0x1c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e70 sbid 0x0)) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd2ee0 blob([0x1c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e70 loaded (sbid 0x2814 ref_map(0x1c3000~2000=1)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e83f95a40 blob([0x1c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2e70 loaded (sbid 0x2814 ref_map(0x1c3000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.blob(0x558e83f95a40) get_ref 0x0~2000 Blob(0x558e83f95a40 blob([0x1c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2e70 loaded (sbid 0x2814 ref_map(0x1c3000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.blob(0x558e83f95a40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x0~2000: 0x0~2000 Blob(0x558e83f95a40 blob([0x1c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e70 loaded (sbid 0x2814 ref_map(0x1c3000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x2000~2000: 0x0~2000 Blob(0x558e83dd2bd0 blob([0x1c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e00 sbid 0x0)) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10261 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd2bd0 blob([0x1c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e00 sbid 0x0)) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd2bd0 blob([0x1c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e00 loaded (sbid 0x2815 ref_map(0x1c5000~2000=1)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e83f949a0 blob([0x1c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2e00 loaded (sbid 0x2815 ref_map(0x1c5000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.blob(0x558e83f949a0) get_ref 0x0~2000 Blob(0x558e83f949a0 blob([0x1c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2e00 loaded (sbid 0x2815 ref_map(0x1c5000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.blob(0x558e83f949a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x2000~2000: 0x0~2000 Blob(0x558e83f949a0 blob([0x1c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e00 loaded (sbid 0x2815 ref_map(0x1c5000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x4000~2000: 0x0~2000 Blob(0x558e83dd30a0 blob([0x1c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3180 sbid 0x0)) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10262 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd30a0 blob([0x1c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3180 sbid 0x0)) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd30a0 blob([0x1c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3180 loaded (sbid 0x2816 ref_map(0x1c7000~2000=1)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e83eded90 blob([0x1c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3180 loaded (sbid 0x2816 ref_map(0x1c7000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.blob(0x558e83eded90) get_ref 0x0~2000 Blob(0x558e83eded90 blob([0x1c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3180 loaded (sbid 0x2816 ref_map(0x1c7000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.blob(0x558e83eded90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x4000~2000: 0x0~2000 Blob(0x558e83eded90 blob([0x1c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3180 loaded (sbid 0x2816 ref_map(0x1c7000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x6000~2000: 0x0~2000 Blob(0x558e83dd2fc0 blob([0x1c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2c40 sbid 0x0)) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10263 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd2fc0 blob([0x1c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2c40 sbid 0x0)) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd2fc0 blob([0x1c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2c40 loaded (sbid 0x2817 ref_map(0x1c9000~2000=1)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e83f94a10 blob([0x1c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2c40 loaded (sbid 0x2817 ref_map(0x1c9000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.blob(0x558e83f94a10) get_ref 0x0~2000 Blob(0x558e83f94a10 blob([0x1c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2c40 loaded (sbid 0x2817 ref_map(0x1c9000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.blob(0x558e83f94a10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x6000~2000: 0x0~2000 Blob(0x558e83f94a10 blob([0x1c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2c40 loaded (sbid 0x2817 ref_map(0x1c9000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x8000~2000: 0x0~2000 Blob(0x558e83dd3030 blob([0x1cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2b60 sbid 0x0)) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10264 2019-12-07T19:34:04.726+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd3030 blob([0x1cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2b60 sbid 0x0)) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd3030 blob([0x1cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2b60 loaded (sbid 0x2818 ref_map(0x1cb000~2000=1)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e83f95c00 blob([0x1cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2b60 loaded (sbid 0x2818 ref_map(0x1cb000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.blob(0x558e83f95c00) get_ref 0x0~2000 Blob(0x558e83f95c00 blob([0x1cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2b60 loaded (sbid 0x2818 ref_map(0x1cb000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.blob(0x558e83f95c00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x8000~2000: 0x0~2000 Blob(0x558e83f95c00 blob([0x1cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2b60 loaded (sbid 0x2818 ref_map(0x1cb000~2000=2)))) 2019-12-07T19:34:04.726+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0xa000~2000: 0x0~2000 Blob(0x558e83dd3110 blob([0x1cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2af0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10265 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd3110 blob([0x1cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2af0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd3110 blob([0x1cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2af0 loaded (sbid 0x2819 ref_map(0x1cd000~2000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e83f94690 blob([0x1cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2af0 loaded (sbid 0x2819 ref_map(0x1cd000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83f94690) get_ref 0x0~2000 Blob(0x558e83f94690 blob([0x1cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2af0 loaded (sbid 0x2819 ref_map(0x1cd000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83f94690) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0xa000~2000: 0x0~2000 Blob(0x558e83f94690 blob([0x1cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2af0 loaded (sbid 0x2819 ref_map(0x1cd000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0xc000~2000: 0x0~2000 Blob(0x558e83dd23f0 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2380 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10266 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd23f0 blob([0x1cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2380 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd23f0 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2380 loaded (sbid 0x281a ref_map(0x1cf000~2000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e840145b0 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2380 loaded (sbid 0x281a ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e840145b0) get_ref 0x0~2000 Blob(0x558e840145b0 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2380 loaded (sbid 0x281a ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e840145b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0xc000~2000: 0x0~2000 Blob(0x558e840145b0 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2380 loaded (sbid 0x281a ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0xe000~2000: 0x0~2000 Blob(0x558e83dd2310 blob([0x1d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd22a0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10267 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd2310 blob([0x1d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd22a0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd2310 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd22a0 loaded (sbid 0x281b ref_map(0x1d1000~2000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e840143f0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd22a0 loaded (sbid 0x281b ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e840143f0) get_ref 0x0~2000 Blob(0x558e840143f0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd22a0 loaded (sbid 0x281b ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e840143f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0xe000~2000: 0x0~2000 Blob(0x558e840143f0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd22a0 loaded (sbid 0x281b ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x10000~2000: 0x0~2000 Blob(0x558e83dd2a80 blob([0x1d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd29a0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10268 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd2a80 blob([0x1d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd29a0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd2a80 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd29a0 loaded (sbid 0x281c ref_map(0x1d3000~2000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e84014850 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd29a0 loaded (sbid 0x281c ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e84014850) get_ref 0x0~2000 Blob(0x558e84014850 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd29a0 loaded (sbid 0x281c ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e84014850) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x10000~2000: 0x0~2000 Blob(0x558e84014850 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd29a0 loaded (sbid 0x281c ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x12000~2000: 0x0~2000 Blob(0x558e83dd3260 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd32d0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10269 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd3260 blob([0x1d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd32d0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd3260 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd32d0 loaded (sbid 0x281d ref_map(0x1d5000~2000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e84014150 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd32d0 loaded (sbid 0x281d ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e84014150) get_ref 0x0~2000 Blob(0x558e84014150 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd32d0 loaded (sbid 0x281d ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e84014150) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x12000~2000: 0x0~2000 Blob(0x558e84014150 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd32d0 loaded (sbid 0x281d ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x14000~2000: 0x0~2000 Blob(0x558e83dd3340 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd33b0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10270 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd3340 blob([0x1d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd33b0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd3340 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd33b0 loaded (sbid 0x281e ref_map(0x1d7000~2000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e84014930 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd33b0 loaded (sbid 0x281e ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e84014930) get_ref 0x0~2000 Blob(0x558e84014930 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd33b0 loaded (sbid 0x281e ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e84014930) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x14000~2000: 0x0~2000 Blob(0x558e84014930 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd33b0 loaded (sbid 0x281e ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x16000~2000: 0x0~2000 Blob(0x558e83dd3420 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3490 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10271 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd3420 blob([0x1d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3490 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd3420 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3490 loaded (sbid 0x281f ref_map(0x1d9000~2000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e841a2a10 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3490 loaded (sbid 0x281f ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e841a2a10) get_ref 0x0~2000 Blob(0x558e841a2a10 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3490 loaded (sbid 0x281f ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e841a2a10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x16000~2000: 0x0~2000 Blob(0x558e841a2a10 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3490 loaded (sbid 0x281f ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x18000~2000: 0x0~2000 Blob(0x558e83dd3500 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3570 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10272 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd3500 blob([0x1db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3570 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd3500 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3570 loaded (sbid 0x2820 ref_map(0x1db000~2000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e841a29a0 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3570 loaded (sbid 0x2820 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e841a29a0) get_ref 0x0~2000 Blob(0x558e841a29a0 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3570 loaded (sbid 0x2820 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e841a29a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x18000~2000: 0x0~2000 Blob(0x558e841a29a0 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3570 loaded (sbid 0x2820 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x1a000~2000: 0x0~2000 Blob(0x558e83dd35e0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3650 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10273 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd35e0 blob([0x1dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3650 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd35e0 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3650 loaded (sbid 0x2821 ref_map(0x1dd000~2000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e83e6ff10 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3650 loaded (sbid 0x2821 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83e6ff10) get_ref 0x0~2000 Blob(0x558e83e6ff10 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3650 loaded (sbid 0x2821 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83e6ff10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558e83e6ff10 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3650 loaded (sbid 0x2821 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x1c000~2000: 0x0~2000 Blob(0x558e83dd36c0 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3730 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10274 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd36c0 blob([0x1df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3730 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd36c0 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3730 loaded (sbid 0x2822 ref_map(0x1df000~2000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e83e6fdc0 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3730 loaded (sbid 0x2822 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83e6fdc0) get_ref 0x0~2000 Blob(0x558e83e6fdc0 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3730 loaded (sbid 0x2822 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83e6fdc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x1c000~2000: 0x0~2000 Blob(0x558e83e6fdc0 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3730 loaded (sbid 0x2822 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x1e000~2000: 0x0~2000 Blob(0x558e83dd37a0 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3810 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10275 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd37a0 blob([0x1e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3810 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd37a0 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3810 loaded (sbid 0x2823 ref_map(0x1e1000~2000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e83e6fa40 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3810 loaded (sbid 0x2823 ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83e6fa40) get_ref 0x0~2000 Blob(0x558e83e6fa40 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3810 loaded (sbid 0x2823 ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83e6fa40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x1e000~2000: 0x0~2000 Blob(0x558e83e6fa40 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3810 loaded (sbid 0x2823 ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x20000~2000: 0x0~2000 Blob(0x558e83dd3880 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd38f0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10276 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd3880 blob([0x1e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd38f0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd3880 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd38f0 loaded (sbid 0x2824 ref_map(0x1e3000~2000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e83e6fab0 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd38f0 loaded (sbid 0x2824 ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83e6fab0) get_ref 0x0~2000 Blob(0x558e83e6fab0 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd38f0 loaded (sbid 0x2824 ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83e6fab0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x20000~2000: 0x0~2000 Blob(0x558e83e6fab0 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd38f0 loaded (sbid 0x2824 ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x22000~2000: 0x0~2000 Blob(0x558e83dd3960 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd39d0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10277 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd3960 blob([0x1e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd39d0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd3960 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd39d0 loaded (sbid 0x2825 ref_map(0x1e5000~2000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e83e6fb20 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd39d0 loaded (sbid 0x2825 ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83e6fb20) get_ref 0x0~2000 Blob(0x558e83e6fb20 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd39d0 loaded (sbid 0x2825 ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83e6fb20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x22000~2000: 0x0~2000 Blob(0x558e83e6fb20 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd39d0 loaded (sbid 0x2825 ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x24000~2000: 0x0~2000 Blob(0x558e83dd3a40 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3ab0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10278 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd3a40 blob([0x1e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3ab0 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd3a40 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3ab0 loaded (sbid 0x2826 ref_map(0x1e7000~2000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e83e6fc70 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3ab0 loaded (sbid 0x2826 ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83e6fc70) get_ref 0x0~2000 Blob(0x558e83e6fc70 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3ab0 loaded (sbid 0x2826 ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83e6fc70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x24000~2000: 0x0~2000 Blob(0x558e83e6fc70 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3ab0 loaded (sbid 0x2826 ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup src 0x26000~1000: 0x0~1000 Blob(0x558e83dd3b20 blob([0x1e9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83dd3b90 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10279 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared Blob(0x558e83dd3b20 blob([0x1e9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83dd3b90 sbid 0x0)) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) make_blob_shared now Blob(0x558e83dd3b20 blob([0x1e9000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83dd3b90 loaded (sbid 0x2827 ref_map(0x1e9000~1000=1)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup new Blob(0x558e83e6fce0 blob([0x1e9000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3b90 loaded (sbid 0x2827 ref_map(0x1e9000~1000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83e6fce0) get_ref 0x0~1000 Blob(0x558e83e6fce0 blob([0x1e9000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3b90 loaded (sbid 0x2827 ref_map(0x1e9000~1000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.blob(0x558e83e6fce0) get_ref init 0x1000, 1000 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dup dst 0x26000~1000: 0x0~1000 Blob(0x558e83e6fce0 blob([0x1e9000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83dd3b90 loaded (sbid 0x2827 ref_map(0x1e9000~1000=2)))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e81a03788) dirty_range mark inline shard dirty 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _clone 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# -> 0#1:41df217a:::smithi13140796-39:4# = 0 2019-12-07T19:34:04.727+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rmattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# snapset 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rmattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# snapset = 0 2019-12-07T19:34:04.727+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# 1 keys 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# 1 keys = 0 2019-12-07T19:34:04.727+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# hinfo_key (42 bytes) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:41df217a:::smithi13140796-39:head#c key 0x80800000000000000141df21'z!smithi13140796-39!='0xfffffffffffffffe000000000000000c'o' 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:34:04.727+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# -> 0#1:41df217a:::smithi13140796-39:head#c 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# -> 0#1:41df217a:::smithi13140796-39:head#c = 0 2019-12-07T19:34:04.727+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1129 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# = 0 2019-12-07T19:34:04.727+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2 keys 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2 keys = 0 2019-12-07T19:34:04.727+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key (42 bytes) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:04.727+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.28BF48E5.4.smithi13140796-39..' <- OBJ_.0_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.727+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.0_0000000000000001.28BF48E5.4.smithi13140796-39..' <- SNA_1_0000000000000004_.0_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.727+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000027.00000000000000000011' <- 0000000027.00000000000000000011 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000027.00000000000000000012' <- 0000000027.00000000000000000012 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417a300 cost 8084 (1 ios * 4000 + 4084 bytes) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417a300 onodes 0x558e81a03680,0x558e83cb3680,0x558e83cb3900 shared_blobs 0x558e83dd22a0,0x558e83dd2380,0x558e83dd29a0,0x558e83dd2af0,0x558e83dd2b60,0x558e83dd2c40,0x558e83dd2e00,0x558e83dd2e70,0x558e83dd3180,0x558e83dd32d0,0x558e83dd33b0,0x558e83dd3490,0x558e83dd3570,0x558e83dd3650,0x558e83dd3730,0x558e83dd3810,0x558e83dd38f0,0x558e83dd39d0,0x558e83dd3ab0,0x558e83dd3b90 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e81a03788) update 0#1:41df217a:::smithi13140796-39:4# 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e81a03788) update inline shard 540 bytes from 20 extents 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:41df217a:::smithi13140796-39:4# is 958 (412 bytes onode + 2 bytes spanning blobs + 544 bytes inline extents) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3788) update 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3788) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:41df217a:::smithi13140796-39:head# is 475 (467 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) update 0#1:41df217a:::smithi13140796-39:head#c 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) update inline shard 540 bytes from 20 extents 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:41df217a:::smithi13140796-39:head#c is 1031 (485 bytes onode + 2 bytes spanning blobs + 544 bytes inline extents) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281b is 11 SharedBlob(0x558e83dd22a0 loaded (sbid 0x281b ref_map(0x1d1000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281a is 11 SharedBlob(0x558e83dd2380 loaded (sbid 0x281a ref_map(0x1cf000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281c is 11 SharedBlob(0x558e83dd29a0 loaded (sbid 0x281c ref_map(0x1d3000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2819 is 11 SharedBlob(0x558e83dd2af0 loaded (sbid 0x2819 ref_map(0x1cd000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2818 is 11 SharedBlob(0x558e83dd2b60 loaded (sbid 0x2818 ref_map(0x1cb000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2817 is 11 SharedBlob(0x558e83dd2c40 loaded (sbid 0x2817 ref_map(0x1c9000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2815 is 11 SharedBlob(0x558e83dd2e00 loaded (sbid 0x2815 ref_map(0x1c5000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2814 is 11 SharedBlob(0x558e83dd2e70 loaded (sbid 0x2814 ref_map(0x1c3000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2816 is 11 SharedBlob(0x558e83dd3180 loaded (sbid 0x2816 ref_map(0x1c7000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281d is 11 SharedBlob(0x558e83dd32d0 loaded (sbid 0x281d ref_map(0x1d5000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281e is 11 SharedBlob(0x558e83dd33b0 loaded (sbid 0x281e ref_map(0x1d7000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281f is 11 SharedBlob(0x558e83dd3490 loaded (sbid 0x281f ref_map(0x1d9000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2820 is 11 SharedBlob(0x558e83dd3570 loaded (sbid 0x2820 ref_map(0x1db000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2821 is 11 SharedBlob(0x558e83dd3650 loaded (sbid 0x2821 ref_map(0x1dd000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2822 is 11 SharedBlob(0x558e83dd3730 loaded (sbid 0x2822 ref_map(0x1df000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2823 is 11 SharedBlob(0x558e83dd3810 loaded (sbid 0x2823 ref_map(0x1e1000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2824 is 11 SharedBlob(0x558e83dd38f0 loaded (sbid 0x2824 ref_map(0x1e3000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2825 is 11 SharedBlob(0x558e83dd39d0 loaded (sbid 0x2825 ref_map(0x1e5000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2826 is 11 SharedBlob(0x558e83dd3ab0 loaded (sbid 0x2826 ref_map(0x1e7000~2000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2827 is 11 SharedBlob(0x558e83dd3b90 loaded (sbid 0x2827 ref_map(0x1e9000~1000=2))) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417a300 allocated 0x[] released 0x[] 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 prepare 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417a300 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 io_done 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 lua=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] op_applied version 27'12 2019-12-07T19:34:04.727+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5297 < min_flush_size 524288 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] eval_repop repgather(0x558e81123d40 27'12 rep_tid=23 committed?=0 r=0) 2019-12-07T19:34:04.727+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edc420 finish 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.727+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03680 had 1 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3680 had 1 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3900 had 1 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:04.727+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5320 < min_flush_size 524288 2019-12-07T19:34:04.727+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5320 < min_flush_size 524288 2019-12-07T19:34:04.727+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.727+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc8a36~14c8 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.727+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc8a36 0xc8a36~14c8 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x98a36 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xa36 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xefe and padding block with 0x102 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xc9efe 2019-12-07T19:34:04.727+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.727+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.727+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000165217s (0.000000306s flush + 0.000164911s kv commit) 2019-12-07T19:34:04.727+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.727+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.727+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417a300> 2019-12-07T19:34:04.727+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.727+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 kv_submitted 2019-12-07T19:34:04.727+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417a300 2019-12-07T19:34:04.727+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 finishing 2019-12-07T19:34:04.727+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417a300 onodes 0x558e81a03680,0x558e83cb3680,0x558e83cb3900 2019-12-07T19:34:04.727+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a300 done 2019-12-07T19:34:04.727+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:04.727+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417a300 [] 2019-12-07T19:34:04.727+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.727+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.727+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.728+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 89 ==== MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e811f7800 con 0x558e818d7000 2019-12-07T19:34:04.728+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edc580 prio 196 cost 0 latency 0.000015 epoch 27 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.728+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) 2019-12-07T19:34:04.728+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.728+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) queued 2019-12-07T19:34:04.728+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.728+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.728+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edc580 prio 196 cost 0 latency 0.000082 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 pg pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.728+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.728+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edc580 finish 2019-12-07T19:34:04.728+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.728+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 44 ==== MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84053000 con 0x558e818d6800 2019-12-07T19:34:04.728+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e819438c0 prio 196 cost 0 latency 0.000014 epoch 27 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.728+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) 2019-12-07T19:34:04.728+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 26 ==== MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e83ca8600 con 0x558e818fe000 2019-12-07T19:34:04.728+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e841769a0 prio 196 cost 0 latency 0.000008 epoch 27 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.728+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) 2019-12-07T19:34:04.728+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.728+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) queued 2019-12-07T19:34:04.728+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.728+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.728+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e819438c0 prio 196 cost 0 latency 0.000081 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 pg pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.728+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.728+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e819438c0 finish 2019-12-07T19:34:04.728+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.728+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) queued 2019-12-07T19:34:04.728+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.728+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.728+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e841769a0 prio 196 cost 0 latency 0.000090 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 pg pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.728+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.728+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e841769a0 finish 2019-12-07T19:34:04.728+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.728+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.729+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 85 ==== MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e81239c00 con 0x558e818fe400 2019-12-07T19:34:04.729+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e83edd4a0 prio 196 cost 0 latency 0.000019 epoch 27 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.729+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) 2019-12-07T19:34:04.729+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.729+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) queued 2019-12-07T19:34:04.729+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.729+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.729+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edd4a0 prio 196 cost 0 latency 0.000048 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 pg pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.729+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.729+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edd4a0 finish 2019-12-07T19:34:04.729+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.729+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 158 ==== MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e81964a00 con 0x558e818d7c00 2019-12-07T19:34:04.729+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edd340 prio 196 cost 0 latency 0.000016 epoch 27 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.729+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) queued 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.729+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edd340 prio 196 cost 0 latency 0.000051 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 pg pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.729+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=23, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.729+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] handle_sub_write_reply Calling on_all_commit on Op(1:41df217a:::smithi13140796-39:head v=27'12 tt=0'0 tid=23 reqid=client.4540.0:231 client_op=osd_op(client.4540.0:231 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 4=[4] ondisk+write+peerstat_old+known_if_redirected e27) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:4=[],1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:34:04.729+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 luod=23'10 crt=23'10 lcod 21'9 mlcod 0'0 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] repop_all_committed: repop tid 23 all committed 2019-12-07T19:34:04.729+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] eval_repop repgather(0x558e81123d40 27'12 rep_tid=23 committed?=1 r=0) 2019-12-07T19:34:04.729+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] commit: repgather(0x558e81123d40 27'12 rep_tid=23 committed?=1 r=0) 2019-12-07T19:34:04.729+0000 7fe2c6a64700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] log_op_stats osd_op(client.4540.0:231 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 4=[4] ondisk+write+peerstat_old+known_if_redirected e27) v8 inb 0 outb 0 lat 0.004263 2019-12-07T19:34:04.729+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] sending reply on osd_op(client.4540.0:231 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 4=[4] ondisk+write+peerstat_old+known_if_redirected e27) v8 0x558e84243400 2019-12-07T19:34:04.729+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(231 smithi13140796-39 [delete] v27'12 uv12 ondisk = 0) v8 -- 0x558e84243400 con 0x558e83d68000 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 0(0) primary objects 4 missing 0 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 1(2) objects 3 missing 0 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 3(5) objects 3 missing 0 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 4(1) objects 3 missing 0 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 5(3) objects 0 missing 0 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 0 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats shard 7(4) objects 0 missing 0 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1)=4} 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:04.729+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats missing shard 7(4) missing= 0 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] publish_stats_to_osd 27:66 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] removing repgather(0x558e81123d40 27'12 rep_tid=23 committed?=1 r=0) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] q front is repgather(0x558e81123d40 27'12 rep_tid=23 committed?=1 r=0) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 15 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] do_osd_op_effects client.4540 con 0x558e83d68000 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] remove_repop repgather(0x558e81123d40 27'12 rep_tid=23 committed?=1 r=0) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_finish_rmw: Op(1:41df217a:::smithi13140796-39:head v=27'12 tt=0'0 tid=23 reqid=client.4540.0:231 client_op=osd_op(client.4540.0:231 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 4=[4] ondisk+write+peerstat_old+known_if_redirected e27) roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:4=[],1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_reads_to_commit: starting commit on Op(1:41df217a:::smithi13140796-39:head v=0'0 tt=0'0 tid=24 reqid=client.4540.0:231 roll_forward_to=27'12 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_reads_to_commit: op: Op(1:41df217a:::smithi13140796-39:head v=0'0 tt=0'0 tid=24 reqid=client.4540.0:231 roll_forward_to=27'12 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:04.730+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 27/26 ECSubWrite(tid=24, reqid=client.4540.0:231, at_version=0'0, trim_to=0'0, roll_forward_to=27'12)) v2 -- 0x558e819da000 con 0x558e818d7c00 2019-12-07T19:34:04.730+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 27/26 ECSubWrite(tid=24, reqid=client.4540.0:231, at_version=0'0, trim_to=0'0, roll_forward_to=27'12)) v2 -- 0x558e84041800 con 0x558e818d7000 2019-12-07T19:34:04.730+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 27/26 ECSubWrite(tid=24, reqid=client.4540.0:231, at_version=0'0, trim_to=0'0, roll_forward_to=27'12)) v2 -- 0x558e83cbe800 con 0x558e818fe400 2019-12-07T19:34:04.730+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s3 27/26 ECSubWrite(tid=24, reqid=client.4540.0:231, at_version=0'0, trim_to=0'0, roll_forward_to=27'12)) v2 -- 0x558e819cf000 con 0x558e818d6800 2019-12-07T19:34:04.730+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpWrite(1.2s4 27/26 ECSubWrite(tid=24, reqid=client.4540.0:231, at_version=0'0, trim_to=0'0, roll_forward_to=27'12)) v2 -- 0x558e819d0800 con 0x558e818fe000 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=23'10 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] append_log log((0'0,27'12], crt=23'10) [] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] rollforward: entry=27'11 (21'5) clone 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:33:58.731948+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] rollforward: entry=27'12 (21'5) modify 1:41df217a:::smithi13140796-39:head by client.4540.0:231 2019-12-07T19:34:04.726120+0000 0 ObjectCleanRegions clean_offsets: [638976~18446744073708912639], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] append_log approx pg log length = 12 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.730+0000 7fe2c6a64700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e8417b500 seq 28 2019-12-07T19:34:04.730+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.2s0_head 0#1:41df217a:::smithi13140796-39:head#c onode 0x558e83cb3900 txc 0x558e8417b500 2019-12-07T19:34:04.730+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.2s0_head 0#1:41df217a:::smithi13140796-39:head#c 0x0 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd2ee0) put_ref 0x0~2000 Blob(0x558e83dd2ee0 blob([0x1c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e70 loaded (sbid 0x2814 ref_map(0x1c3000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd2bd0) put_ref 0x0~2000 Blob(0x558e83dd2bd0 blob([0x1c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e00 loaded (sbid 0x2815 ref_map(0x1c5000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd30a0) put_ref 0x0~2000 Blob(0x558e83dd30a0 blob([0x1c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3180 loaded (sbid 0x2816 ref_map(0x1c7000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd2fc0) put_ref 0x0~2000 Blob(0x558e83dd2fc0 blob([0x1c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2c40 loaded (sbid 0x2817 ref_map(0x1c9000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd3030) put_ref 0x0~2000 Blob(0x558e83dd3030 blob([0x1cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2b60 loaded (sbid 0x2818 ref_map(0x1cb000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd3110) put_ref 0x0~2000 Blob(0x558e83dd3110 blob([0x1cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2af0 loaded (sbid 0x2819 ref_map(0x1cd000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd23f0) put_ref 0x0~2000 Blob(0x558e83dd23f0 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2380 loaded (sbid 0x281a ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd2310) put_ref 0x0~2000 Blob(0x558e83dd2310 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd22a0 loaded (sbid 0x281b ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd2a80) put_ref 0x0~2000 Blob(0x558e83dd2a80 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd29a0 loaded (sbid 0x281c ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd3260) put_ref 0x0~2000 Blob(0x558e83dd3260 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd32d0 loaded (sbid 0x281d ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd3340) put_ref 0x0~2000 Blob(0x558e83dd3340 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd33b0 loaded (sbid 0x281e ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd3420) put_ref 0x0~2000 Blob(0x558e83dd3420 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3490 loaded (sbid 0x281f ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd3500) put_ref 0x0~2000 Blob(0x558e83dd3500 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3570 loaded (sbid 0x2820 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd35e0) put_ref 0x0~2000 Blob(0x558e83dd35e0 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3650 loaded (sbid 0x2821 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd36c0) put_ref 0x0~2000 Blob(0x558e83dd36c0 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3730 loaded (sbid 0x2822 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd37a0) put_ref 0x0~2000 Blob(0x558e83dd37a0 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3810 loaded (sbid 0x2823 ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd3880) put_ref 0x0~2000 Blob(0x558e83dd3880 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd38f0 loaded (sbid 0x2824 ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd3960) put_ref 0x0~2000 Blob(0x558e83dd3960 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd39d0 loaded (sbid 0x2825 ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd3a40) put_ref 0x0~2000 Blob(0x558e83dd3a40 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3ab0 loaded (sbid 0x2826 ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.blob(0x558e83dd3b20) put_ref 0x0~1000 Blob(0x558e83dd3b20 blob([0x1e9000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83dd3b90 loaded (sbid 0x2827 ref_map(0x1e9000~1000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3a08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e83dd2ee0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd2e70 loaded (sbid 0x2814 ref_map(0x1c3000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1c3000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd2e70 loaded (sbid 0x2814 ref_map(0x1c3000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e83dd2bd0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd2e00 loaded (sbid 0x2815 ref_map(0x1c5000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1c5000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd2e00 loaded (sbid 0x2815 ref_map(0x1c5000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e83dd30a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd3180 loaded (sbid 0x2816 ref_map(0x1c7000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1c7000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd3180 loaded (sbid 0x2816 ref_map(0x1c7000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e83dd2fc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd2c40 loaded (sbid 0x2817 ref_map(0x1c9000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1c9000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd2c40 loaded (sbid 0x2817 ref_map(0x1c9000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e83dd3030 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd2b60 loaded (sbid 0x2818 ref_map(0x1cb000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1cb000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd2b60 loaded (sbid 0x2818 ref_map(0x1cb000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e83dd3110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd2af0 loaded (sbid 0x2819 ref_map(0x1cd000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1cd000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd2af0 loaded (sbid 0x2819 ref_map(0x1cd000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e83dd23f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd2380 loaded (sbid 0x281a ref_map(0x1cf000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1cf000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd2380 loaded (sbid 0x281a ref_map(0x1cf000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558e83dd2310 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd22a0 loaded (sbid 0x281b ref_map(0x1d1000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1d1000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd22a0 loaded (sbid 0x281b ref_map(0x1d1000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558e83dd2a80 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd29a0 loaded (sbid 0x281c ref_map(0x1d3000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1d3000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd29a0 loaded (sbid 0x281c ref_map(0x1d3000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558e83dd3260 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd32d0 loaded (sbid 0x281d ref_map(0x1d5000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1d5000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd32d0 loaded (sbid 0x281d ref_map(0x1d5000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558e83dd3340 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd33b0 loaded (sbid 0x281e ref_map(0x1d7000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1d7000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd33b0 loaded (sbid 0x281e ref_map(0x1d7000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558e83dd3420 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd3490 loaded (sbid 0x281f ref_map(0x1d9000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1d9000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd3490 loaded (sbid 0x281f ref_map(0x1d9000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558e83dd3500 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd3570 loaded (sbid 0x2820 ref_map(0x1db000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1db000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd3570 loaded (sbid 0x2820 ref_map(0x1db000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558e83dd35e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd3650 loaded (sbid 0x2821 ref_map(0x1dd000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1dd000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd3650 loaded (sbid 0x2821 ref_map(0x1dd000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558e83dd36c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd3730 loaded (sbid 0x2822 ref_map(0x1df000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1df000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd3730 loaded (sbid 0x2822 ref_map(0x1df000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558e83dd37a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd3810 loaded (sbid 0x2823 ref_map(0x1e1000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1e1000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd3810 loaded (sbid 0x2823 ref_map(0x1e1000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558e83dd3880 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd38f0 loaded (sbid 0x2824 ref_map(0x1e3000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1e3000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd38f0 loaded (sbid 0x2824 ref_map(0x1e3000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558e83dd3960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd39d0 loaded (sbid 0x2825 ref_map(0x1e5000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1e5000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd39d0 loaded (sbid 0x2825 ref_map(0x1e5000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558e83dd3a40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd3ab0 loaded (sbid 0x2826 ref_map(0x1e7000~2000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1e7000~2000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd3ab0 loaded (sbid 0x2826 ref_map(0x1e7000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x26000~1000: 0x0~1000 Blob(0x558e83dd3b20 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558e83dd3b90 loaded (sbid 0x2827 ref_map(0x1e9000~1000=2)))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1e9000~1000] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd3b90 loaded (sbid 0x2827 ref_map(0x1e9000~1000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove gen and maybe_unshared_blobs 0x558e83dd22a0,0x558e83dd2380,0x558e83dd29a0,0x558e83dd2af0,0x558e83dd2b60,0x558e83dd2c40,0x558e83dd2e00,0x558e83dd2e70,0x558e83dd3180,0x558e83dd32d0,0x558e83dd33b0,0x558e83dd3490,0x558e83dd3570,0x558e83dd3650,0x558e83dd3730,0x558e83dd3810,0x558e83dd38f0,0x558e83dd39d0,0x558e83dd3ab0,0x558e83dd3b90 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove checking for unshareable blobs on 0x558e83cb3680 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.2s0_head 0#1:41df217a:::smithi13140796-39:head#c = 0 2019-12-07T19:34:04.730+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417b500 cost 5574 (1 ios * 4000 + 1574 bytes) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417b500 onodes shared_blobs 0x558e83dd22a0,0x558e83dd2380,0x558e83dd29a0,0x558e83dd2af0,0x558e83dd2b60,0x558e83dd2c40,0x558e83dd2e00,0x558e83dd2e70,0x558e83dd3180,0x558e83dd32d0,0x558e83dd33b0,0x558e83dd3490,0x558e83dd3570,0x558e83dd3650,0x558e83dd3730,0x558e83dd3810,0x558e83dd38f0,0x558e83dd39d0,0x558e83dd3ab0,0x558e83dd3b90 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281b is 11 SharedBlob(0x558e83dd22a0 loaded (sbid 0x281b ref_map(0x1d1000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281a is 11 SharedBlob(0x558e83dd2380 loaded (sbid 0x281a ref_map(0x1cf000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281c is 11 SharedBlob(0x558e83dd29a0 loaded (sbid 0x281c ref_map(0x1d3000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2819 is 11 SharedBlob(0x558e83dd2af0 loaded (sbid 0x2819 ref_map(0x1cd000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2818 is 11 SharedBlob(0x558e83dd2b60 loaded (sbid 0x2818 ref_map(0x1cb000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2817 is 11 SharedBlob(0x558e83dd2c40 loaded (sbid 0x2817 ref_map(0x1c9000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2815 is 11 SharedBlob(0x558e83dd2e00 loaded (sbid 0x2815 ref_map(0x1c5000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2814 is 11 SharedBlob(0x558e83dd2e70 loaded (sbid 0x2814 ref_map(0x1c3000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2816 is 11 SharedBlob(0x558e83dd3180 loaded (sbid 0x2816 ref_map(0x1c7000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281d is 11 SharedBlob(0x558e83dd32d0 loaded (sbid 0x281d ref_map(0x1d5000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281e is 11 SharedBlob(0x558e83dd33b0 loaded (sbid 0x281e ref_map(0x1d7000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281f is 11 SharedBlob(0x558e83dd3490 loaded (sbid 0x281f ref_map(0x1d9000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2820 is 11 SharedBlob(0x558e83dd3570 loaded (sbid 0x2820 ref_map(0x1db000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2821 is 11 SharedBlob(0x558e83dd3650 loaded (sbid 0x2821 ref_map(0x1dd000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2822 is 11 SharedBlob(0x558e83dd3730 loaded (sbid 0x2822 ref_map(0x1df000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2823 is 11 SharedBlob(0x558e83dd3810 loaded (sbid 0x2823 ref_map(0x1e1000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2824 is 11 SharedBlob(0x558e83dd38f0 loaded (sbid 0x2824 ref_map(0x1e3000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2825 is 11 SharedBlob(0x558e83dd39d0 loaded (sbid 0x2825 ref_map(0x1e5000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2826 is 11 SharedBlob(0x558e83dd3ab0 loaded (sbid 0x2826 ref_map(0x1e7000~2000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2827 is 11 SharedBlob(0x558e83dd3b90 loaded (sbid 0x2827 ref_map(0x1e9000~1000=1))) 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417b500 allocated 0x[] released 0x[] 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 prepare 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417b500 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 io_done 2019-12-07T19:34:04.730+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edd340 finish 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.730+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.730+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.730+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1830 < min_flush_size 524288 2019-12-07T19:34:04.730+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:04.730+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3900 had 1 2019-12-07T19:34:04.730+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1853 < min_flush_size 524288 2019-12-07T19:34:04.730+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1853 < min_flush_size 524288 2019-12-07T19:34:04.730+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.730+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xc9efe~73d to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.730+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xc9efe 0xc9efe~73d to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.730+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.730+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x99efe 2019-12-07T19:34:04.730+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xefe 2019-12-07T19:34:04.730+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.730+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x63b and padding block with 0x9c5 2019-12-07T19:34:04.730+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xca63b 2019-12-07T19:34:04.730+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.730+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.730+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.730+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.730+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000247480s (0.000000100s flush + 0.000247380s kv commit) 2019-12-07T19:34:04.730+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417b500> 2019-12-07T19:34:04.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.730+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 kv_submitted 2019-12-07T19:34:04.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417b500 2019-12-07T19:34:04.730+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 finishing 2019-12-07T19:34:04.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417b500 onodes 2019-12-07T19:34:04.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b500 done 2019-12-07T19:34:04.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:04.730+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417b500 [] 2019-12-07T19:34:04.730+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.730+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.730+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.730+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.732+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 45 ==== MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e8118f400 con 0x558e818d6800 2019-12-07T19:34:04.732+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e84176b00 prio 196 cost 0 latency 0.000018 epoch 27 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.732+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) 2019-12-07T19:34:04.732+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.732+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) queued 2019-12-07T19:34:04.732+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.732+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.732+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e84176b00 prio 196 cost 0 latency 0.000228 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 pg pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.732+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.732+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e84176b00 finish 2019-12-07T19:34:04.732+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.732+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 27 ==== MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028800 con 0x558e818fe000 2019-12-07T19:34:04.732+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e84176c60 prio 196 cost 0 latency 0.000011 epoch 27 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.732+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) 2019-12-07T19:34:04.732+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.732+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) queued 2019-12-07T19:34:04.732+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.732+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.732+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e84176c60 prio 196 cost 0 latency 0.000347 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 pg pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.732+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.732+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e84176c60 finish 2019-12-07T19:34:04.732+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.733+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 90 ==== MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e81964c00 con 0x558e818d7000 2019-12-07T19:34:04.733+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edd760 prio 196 cost 0 latency 0.000017 epoch 27 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.733+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) 2019-12-07T19:34:04.733+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.733+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) queued 2019-12-07T19:34:04.733+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.733+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.733+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edd760 prio 196 cost 0 latency 0.000202 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 pg pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.733+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.733+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e83edd760 finish 2019-12-07T19:34:04.733+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.733+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 86 ==== MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818a9c00 con 0x558e818fe400 2019-12-07T19:34:04.733+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e83edd600 prio 196 cost 0 latency 0.000014 epoch 27 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.733+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) 2019-12-07T19:34:04.733+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.733+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) queued 2019-12-07T19:34:04.733+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.733+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.733+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 159 ==== MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818a8c00 con 0x558e818d7c00 2019-12-07T19:34:04.733+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84177ce0 prio 196 cost 0 latency 0.000010 epoch 27 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.733+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) 2019-12-07T19:34:04.733+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edd600 prio 196 cost 0 latency 0.000055 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 pg pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.733+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.733+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.733+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) queued 2019-12-07T19:34:04.733+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edd600 finish 2019-12-07T19:34:04.733+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.733+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2) prio 196 cost 0 e27) pg 0x558e81075000 2019-12-07T19:34:04.733+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.733+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e84177ce0 prio 196 cost 0 latency 0.000053 MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 pg pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] 2019-12-07T19:34:04.733+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 27/26 ECSubWriteReply(tid=24, last_complete=27'12, committed=1, applied=1)) v2 2019-12-07T19:34:04.733+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_finish_rmw: Op(1:41df217a:::smithi13140796-39:head v=0'0 tt=0'0 tid=24 reqid=client.4540.0:231 roll_forward_to=27'12 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:04.733+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:04.733+0000 7fe2caa6c700 20 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:04.733+0000 7fe2caa6c700 10 osd.0 27 dequeue_op 0x558e84177ce0 finish 2019-12-07T19:34:04.733+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.739+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 160 ==== MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=116, reqid=client.4540.0:229, at_version=27'21, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2 ==== 1511+0+0 (unknown 0 0 0) 0x558e84041800 con 0x558e818d7c00 2019-12-07T19:34:04.739+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84177b80 prio 127 cost 0 latency 0.000050 epoch 27 MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=116, reqid=client.4540.0:229, at_version=27'21, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2 2019-12-07T19:34:04.739+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=116, reqid=client.4540.0:229, at_version=27'21, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=116, reqid=client.4540.0:229, at_version=27'21, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=116, reqid=client.4540.0:229, at_version=27'21, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) pg 0x558e841a4000 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e84177b80 prio 127 cost 0 latency 0.000273 MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=116, reqid=client.4540.0:229, at_version=27'21, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2 pg pg[1.8s5( v 27'19 lc 21'3 (0'0,27'19] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=6 mbc={} ps=[2~1]] 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'19 lc 21'3 (0'0,27'19] local-lis/les=0/0 n=6 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=6 mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=116, reqid=client.4540.0:229, at_version=27'21, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'19 lc 21'3 (0'0,27'19] local-lis/les=0/0 n=7 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'18 active m=7 mbc={} ps=[2~1]] append_log log((0'0,27'19], crt=27'18) [27'20 (23'14) clone 1:131421e8:::smithi13140796-7:4 by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,27'21 (23'14) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:229 0.000000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 1, new_object: 0] 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'20 lc 21'3 (0'0,27'20] local-lis/les=0/0 n=7 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'19 crt=27'20 active m=7 mbc={} ps=[2~1]] add_log_entry 27'20 (23'14) clone 1:131421e8:::smithi13140796-7:4 by unknown.0.0:0 2019-12-07T19:33:58.699945+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'21 lc 21'3 (0'0,27'21] local-lis/les=0/0 n=7 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'19 crt=27'21 active m=7 mbc={} ps=[2~1]] add_log_entry 27'21 (23'14) modify 1:131421e8:::smithi13140796-7:head by client.4540.0:229 0.000000 0 ObjectCleanRegions clean_offsets: [507904~18446744073709043711], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'21 lc 21'3 (0'0,27'21] local-lis/les=0/0 n=7 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'19 crt=27'21 active m=7 mbc={} ps=[2~1]] append_log approx pg log length = 21 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'21 lc 21'3 (0'0,27'21] local-lis/les=0/0 n=7 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'19 crt=27'21 active m=7 mbc={} ps=[2~1]] append_log transaction_applied = 0 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'21 lc 21'3 (0'0,27'21] local-lis/les=0/0 n=7 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'19 crt=27'21 active m=7 mbc={} ps=[2~1]] append_log 0(5) is async_recovery or backfill target 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.739+0000 7fe2cba6e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'20, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e83e72300 seq 11 2019-12-07T19:34:04.739+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000027.00000000000000000020' <- 0000000027.00000000000000000020 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000027.00000000000000000021' <- 0000000027.00000000000000000021 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.missing/0000000000000001.8C824871.4.smithi13140796-7..' <- missing/0000000000000001.8C824871.4.smithi13140796-7.. 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.missing/0000000000000001.8C824871.head.smithi13140796-7..' <- missing/0000000000000001.8C824871.head.smithi13140796-7.. 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72300 cost 6310 (1 ios * 4000 + 2310 bytes) 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72300 onodes shared_blobs 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72300 allocated 0x[] released 0x[] 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72300 prepare 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72300 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72300 io_done 2019-12-07T19:34:04.739+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.739+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'21 lc 21'3 (0'0,27'21] local-lis/les=0/0 n=7 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'19 crt=27'21 active m=7 mbc={} ps=[2~1]] op_applied version 27'21 2019-12-07T19:34:04.739+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e84177b80 finish 2019-12-07T19:34:04.739+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.739+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2147 < min_flush_size 524288 2019-12-07T19:34:04.739+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.739+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2170 < min_flush_size 524288 2019-12-07T19:34:04.739+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2170 < min_flush_size 524288 2019-12-07T19:34:04.739+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.739+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xca63b~87a to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.739+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xca63b 0xca63b~87a to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.739+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.739+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9a63b 2019-12-07T19:34:04.739+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x63b 2019-12-07T19:34:04.739+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.739+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xeb5 and padding block with 0x14b 2019-12-07T19:34:04.739+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xcaeb5 2019-12-07T19:34:04.739+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.739+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.739+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.739+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.739+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000128533s (0.000000195s flush + 0.000128338s kv commit) 2019-12-07T19:34:04.739+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.739+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.739+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e72300> 2019-12-07T19:34:04.739+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.739+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72300 kv_submitted 2019-12-07T19:34:04.739+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72300 2019-12-07T19:34:04.739+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72300 finishing 2019-12-07T19:34:04.739+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72300 onodes 2019-12-07T19:34:04.739+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72300 done 2019-12-07T19:34:04.739+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.739+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72300 [] 2019-12-07T19:34:04.739+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.739+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.740+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.740+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 27/26 ECSubWriteReply(tid=116, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e84065400 con 0x558e818d7c00 2019-12-07T19:34:04.740+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.741+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 87 ==== MOSDPGPush(1.3s5 27/26 [PushOp(1:c8ce3553:::smithi13140796-29:head, version: 21'6, data_included: [0~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:c8ce3553:::smithi13140796-29:head@21'6, size: 475136, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:475136, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 119846+0+0 (unknown 0 0 0) 0x558e818a8a00 con 0x558e818fe400 2019-12-07T19:34:04.741+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e83edcc60 prio 127 cost 119784 latency 0.000029 epoch 27 MOSDPGPush(1.3s5 27/26 [PushOp(1:c8ce3553:::smithi13140796-29:head, version: 21'6, data_included: [0~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:c8ce3553:::smithi13140796-29:head@21'6, size: 475136, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:475136, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.741+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDPGPush(1.3s5 27/26 [PushOp(1:c8ce3553:::smithi13140796-29:head, version: 21'6, data_included: [0~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:c8ce3553:::smithi13140796-29:head@21'6, size: 475136, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:475136, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 119784 e27) 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDPGPush(1.3s5 27/26 [PushOp(1:c8ce3553:::smithi13140796-29:head, version: 21'6, data_included: [0~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:c8ce3553:::smithi13140796-29:head@21'6, size: 475136, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:475136, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 119784 e27) queued 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDPGPush(1.3s5 27/26 [PushOp(1:c8ce3553:::smithi13140796-29:head, version: 21'6, data_included: [0~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:c8ce3553:::smithi13140796-29:head@21'6, size: 475136, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:475136, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 119784 e27) pg 0x558e81231000 2019-12-07T19:34:04.741+0000 7fe2c6263700 10 osd.0 27 dequeue_op 0x558e83edcc60 prio 127 cost 119784 latency 0.000109 MOSDPGPush(1.3s5 27/26 [PushOp(1:c8ce3553:::smithi13140796-29:head, version: 21'6, data_included: [0~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:c8ce3553:::smithi13140796-29:head@21'6, size: 475136, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:475136, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] 2019-12-07T19:34:04.741+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.3s5 27/26 [PushOp(1:c8ce3553:::smithi13140796-29:head, version: 21'6, data_included: [0~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:c8ce3553:::smithi13140796-29:head@21'6, size: 475136, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:475136, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.741+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] on_local_recover: 1:c8ce3553:::smithi13140796-29:head 2019-12-07T19:34:04.741+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'5 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=3 mbc={} ps=[2~1]] got missing 1:c8ce3553:::smithi13140796-29:head v 21'6 2019-12-07T19:34:04.741+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'6 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=2 mbc={} ps=[2~1]] last_complete now 21'6 log.complete_to 21'7 2019-12-07T19:34:04.741+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.741+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e8195e900 seq 4 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) get_onode oid 5#1:c8ce3553:::smithi13140796-29:head# key 0x858000000000000001c8ce35'S!smithi13140796-29!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) r -2 v.len 0 2019-12-07T19:34:04.741+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 5#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) get_onode oid 5#1:c8ce3553:::smithi13140796-29:head# key 0x858000000000000001c8ce35'S!smithi13140796-29!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) r -2 v.len 0 2019-12-07T19:34:04.741+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1130 2019-12-07T19:34:04.741+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# = 0 2019-12-07T19:34:04.741+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 0x0~1d000 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#1:c8ce3553:::smithi13140796-29:head# 0x0~1d000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.741+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1d000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8195e900 15 blobs 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.741+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.742+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate 0x1d000/1000,1d000,0 2019-12-07T19:34:04.742+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x378000~1d000/1000,1d000,0 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x378000~1d000] 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a2930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a28c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a2930 blob([0x378000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a28c0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e841a2930) get_ref 0x0~2000 Blob(0x558e841a2930 blob([0x378000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a28c0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e841a2930) get_ref init 0x2000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e841a2930 blob([0x378000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a28c0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841a28d8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a2850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f8f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a2850 blob([0x37a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f8f0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e841a2850) get_ref 0x0~2000 Blob(0x558e841a2850 blob([0x37a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f8f0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e841a2850) get_ref init 0x2000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e841a2850 blob([0x37a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f8f0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0f908 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0f1f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0f1f0 blob([0x37c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f180 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f1f0) get_ref 0x0~2000 Blob(0x558e83d0f1f0 blob([0x37c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f180 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83d0f1f0 blob([0x37c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f180 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0f198 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af90a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abd570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af90a0 blob([0x37e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abd570 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e83af90a0) get_ref 0x0~2000 Blob(0x558e83af90a0 blob([0x37e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abd570 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e83af90a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83af90a0 blob([0x37e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abd570 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83abd588 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6fea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6fea0 blob([0x380000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f420 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6fea0) get_ref 0x0~2000 Blob(0x558e83e6fea0 blob([0x380000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f420 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6fea0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83e6fea0 blob([0x380000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6f420 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6f438 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6f650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6f650 blob([0x382000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2d20 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6f650) get_ref 0x0~2000 Blob(0x558e83e6f650 blob([0x382000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2d20 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6f650) get_ref init 0x2000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83e6f650 blob([0x382000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2d20 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83dd2d38 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd2690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd2690 blob([0x384000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2460 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e83dd2690) get_ref 0x0~2000 Blob(0x558e83dd2690 blob([0x384000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2460 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e83dd2690) get_ref init 0x2000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83dd2690 blob([0x384000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2460 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83dd2478 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83edefc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423ccb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83edefc0 blob([0x386000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423ccb0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e83edefc0) get_ref 0x0~2000 Blob(0x558e83edefc0 blob([0x386000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423ccb0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e83edefc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83edefc0 blob([0x386000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ccb0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e8423ccc8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d02540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d029a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d02540 blob([0x388000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d029a0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e83d02540) get_ref 0x0~2000 Blob(0x558e83d02540 blob([0x388000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d029a0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e83d02540) get_ref init 0x2000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83d02540 blob([0x388000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d029a0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d029b8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841dbc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841dbb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841dbc00 blob([0x38a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbb90 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e841dbc00) get_ref 0x0~2000 Blob(0x558e841dbc00 blob([0x38a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbb90 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e841dbc00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e841dbc00 blob([0x38a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbb90 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841dbba8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841dbb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841dbab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841dbb20 blob([0x38c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbab0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e841dbb20) get_ref 0x0~2000 Blob(0x558e841dbb20 blob([0x38c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbab0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e841dbb20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e841dbb20 blob([0x38c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbab0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841dbac8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841dba40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841db9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841dba40 blob([0x38e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db9d0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e841dba40) get_ref 0x0~2000 Blob(0x558e841dba40 blob([0x38e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db9d0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e841dba40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e841dba40 blob([0x38e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db9d0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841db9e8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841db960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841db8f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841db960 blob([0x390000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db8f0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e841db960) get_ref 0x0~2000 Blob(0x558e841db960 blob([0x390000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db8f0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e841db960) get_ref init 0x2000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e841db960 blob([0x390000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db8f0 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841db908 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841db880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e9570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841db880 blob([0x392000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9570 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e841db880) get_ref 0x0~2000 Blob(0x558e841db880 blob([0x392000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9570 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e841db880) get_ref init 0x2000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e841db880 blob([0x392000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9570 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e817e9588 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e817e9730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d37f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e817e9730 blob([0x394000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d37f80 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e817e9730) get_ref 0x0~1000 Blob(0x558e817e9730 blob([0x394000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d37f80 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.blob(0x558e817e9730) get_ref init 0x1000, 1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~1000: 0x0~1000 Blob(0x558e817e9730 blob([0x394000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d37f80 sbid 0x0)) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d37f98 in 0x558e802196c0) _discard 0x0~1000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1d000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dirty_range mark inline shard dirty 2019-12-07T19:34:04.742+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 0x0~1d000 = 0 2019-12-07T19:34:04.742+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 4 keys 2019-12-07T19:34:04.742+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 4 keys = 0 2019-12-07T19:34:04.742+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x0000000000000461'.missing/0000000000000001.3137CAAC.head.smithi13140796-29..' <- missing/0000000000000001.3137CAAC.head.smithi13140796-29.. 2019-12-07T19:34:04.742+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:04.742+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.742+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:04.742+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e900 cost 185021 (16 ios * 4000 + 121021 bytes) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e900 onodes 0x558e84242c80 shared_blobs 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) update 5#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) update inline shard 285 bytes from 15 extents 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:c8ce3553:::smithi13140796-29:head# is 776 (485 bytes onode + 2 bytes spanning blobs + 289 bytes inline extents) 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e900 allocated 0x[378000~1d000] released 0x[] 2019-12-07T19:34:04.742+0000 7fe2c6263700 10 freelist allocate 0x378000~1d000 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 freelist _xor first_key 0x300000 last_key 0x380000 2019-12-07T19:34:04.742+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 prepare 2019-12-07T19:34:04.742+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8195e900 2019-12-07T19:34:04.742+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 161 ==== MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=117, reqid=client.4540.0:236, at_version=27'23, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2 ==== 1440+0+0 (unknown 0 0 0) 0x558e819da000 con 0x558e818d7c00 2019-12-07T19:34:04.742+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84177a20 prio 127 cost 0 latency 0.000032 epoch 27 MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=117, reqid=client.4540.0:236, at_version=27'23, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2 2019-12-07T19:34:04.742+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=117, reqid=client.4540.0:236, at_version=27'23, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.742+0000 7fe2c6263700 10 osd.0 27 dequeue_op 0x558e83edcc60 finish 2019-12-07T19:34:04.742+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.742+0000 7fe2ca26b700 20 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'6 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=2 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.742+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=117, reqid=client.4540.0:236, at_version=27'23, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=117, reqid=client.4540.0:236, at_version=27'23, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) pg 0x558e841a4000 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e84177a20 prio 127 cost 0 latency 0.000938 MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=117, reqid=client.4540.0:236, at_version=27'23, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2 pg pg[1.8s5( v 27'21 lc 21'3 (0'0,27'21] local-lis/les=0/0 n=7 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'21 active m=7 mbc={} ps=[2~1]] 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'21 lc 21'3 (0'0,27'21] local-lis/les=0/0 n=7 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'21 active m=7 mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=117, reqid=client.4540.0:236, at_version=27'23, trim_to=0'0, roll_forward_to=27'18, backfill_or_async_recovery)) v2 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'21 lc 21'3 (0'0,27'21] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'21 active m=8 mbc={} ps=[2~1]] append_log log((0'0,27'21], crt=27'21) [27'22 (21'6) clone 1:15219530:::smithi13140796-21:4 by unknown.0.0:0 2019-12-07T19:33:58.668552+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,27'23 (21'6) modify 1:15219530:::smithi13140796-21:head by client.4540.0:236 2019-12-07T19:34:04.740015+0000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 0, new_object: 0] 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'22 lc 21'3 (0'0,27'22] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'21 crt=27'22 active m=8 mbc={} ps=[2~1]] add_log_entry 27'22 (21'6) clone 1:15219530:::smithi13140796-21:4 by unknown.0.0:0 2019-12-07T19:33:58.668552+0000 0 snaps [4] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'21 crt=27'23 active m=8 mbc={} ps=[2~1]] add_log_entry 27'23 (21'6) modify 1:15219530:::smithi13140796-21:head by client.4540.0:236 2019-12-07T19:34:04.740015+0000 0 ObjectCleanRegions clean_offsets: [344064~18446744073709207551], clean_omap: 0, new_object: 0 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'21 crt=27'23 active m=8 mbc={} ps=[2~1]] append_log approx pg log length = 23 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'21 crt=27'23 active m=8 mbc={} ps=[2~1]] append_log transaction_applied = 0 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'21 crt=27'23 active m=8 mbc={} ps=[2~1]] append_log 0(5) is async_recovery or backfill target 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.743+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 27'22, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e8195e600 seq 12 2019-12-07T19:34:04.743+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000027.00000000000000000022' <- 0000000027.00000000000000000022 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000027.00000000000000000023' <- 0000000027.00000000000000000023 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.missing/0000000000000001.8A489AC0.4.smithi13140796-21..' <- missing/0000000000000001.8A489AC0.4.smithi13140796-21.. 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.missing/0000000000000001.8A489AC0.head.smithi13140796-21..' <- missing/0000000000000001.8A489AC0.head.smithi13140796-21.. 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e600 cost 6242 (1 ios * 4000 + 2242 bytes) 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e600 onodes shared_blobs 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 prepare 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e600 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 io_done 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 lua=27'21 crt=27'23 active m=8 mbc={} ps=[2~1]] op_applied version 27'23 2019-12-07T19:34:04.743+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e84177a20 finish 2019-12-07T19:34:04.743+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.743+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.743+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.743+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2079 < min_flush_size 524288 2019-12-07T19:34:04.743+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.743+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2102 < min_flush_size 524288 2019-12-07T19:34:04.743+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2102 < min_flush_size 524288 2019-12-07T19:34:04.743+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.743+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xcaeb5~836 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.743+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xcaeb5 0xcaeb5~836 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.743+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.743+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9aeb5 2019-12-07T19:34:04.743+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xeb5 2019-12-07T19:34:04.743+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.743+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x6eb and padding block with 0x915 2019-12-07T19:34:04.743+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xcb6eb 2019-12-07T19:34:04.743+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.743+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.743+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.743+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.744+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000254332s (0.000000110s flush + 0.000254222s kv commit) 2019-12-07T19:34:04.744+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e600> 2019-12-07T19:34:04.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.744+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 kv_submitted 2019-12-07T19:34:04.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e600 2019-12-07T19:34:04.744+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 finishing 2019-12-07T19:34:04.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e600 onodes 2019-12-07T19:34:04.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e600 done 2019-12-07T19:34:04.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.744+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e600 [] 2019-12-07T19:34:04.744+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.744+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.744+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 27/26 ECSubWriteReply(tid=117, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e8021f800 con 0x558e818d7c00 2019-12-07T19:34:04.744+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.745+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.746+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 162 ==== MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=118, reqid=client.4540.0:236, at_version=0'0, trim_to=0'0, roll_forward_to=27'23, backfill_or_async_recovery)) v2 ==== 1028+0+0 (unknown 0 0 0) 0x558e84042000 con 0x558e818d7c00 2019-12-07T19:34:04.746+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84176420 prio 127 cost 0 latency 0.000028 epoch 27 MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=118, reqid=client.4540.0:236, at_version=0'0, trim_to=0'0, roll_forward_to=27'23, backfill_or_async_recovery)) v2 2019-12-07T19:34:04.746+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=118, reqid=client.4540.0:236, at_version=0'0, trim_to=0'0, roll_forward_to=27'23, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) 2019-12-07T19:34:04.747+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.747+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=118, reqid=client.4540.0:236, at_version=0'0, trim_to=0'0, roll_forward_to=27'23, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.747+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.747+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=118, reqid=client.4540.0:236, at_version=0'0, trim_to=0'0, roll_forward_to=27'23, backfill_or_async_recovery)) v2) prio 127 cost 0 e27) pg 0x558e841a4000 2019-12-07T19:34:04.747+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e84176420 prio 127 cost 0 latency 0.000629 MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=118, reqid=client.4540.0:236, at_version=0'0, trim_to=0'0, roll_forward_to=27'23, backfill_or_async_recovery)) v2 pg pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=8 mbc={} ps=[2~1]] 2019-12-07T19:34:04.747+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=8 mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.8s5 27/26 ECSubWrite(tid=118, reqid=client.4540.0:236, at_version=0'0, trim_to=0'0, roll_forward_to=27'23, backfill_or_async_recovery)) v2 2019-12-07T19:34:04.747+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=8 mbc={} ps=[2~1]] append_log log((0'0,27'23], crt=27'23) [] 2019-12-07T19:34:04.747+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=8 mbc={} ps=[2~1]] append_log approx pg log length = 23 2019-12-07T19:34:04.747+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=8 mbc={} ps=[2~1]] append_log transaction_applied = 0 2019-12-07T19:34:04.747+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=8 mbc={} ps=[2~1]] append_log 0(5) is async_recovery or backfill target 2019-12-07T19:34:04.747+0000 7fe2cba6e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:04.747+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:04.747+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.747+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e83e72600 seq 13 2019-12-07T19:34:04.747+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.747+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.747+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.747+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72600 cost 5342 (1 ios * 4000 + 1342 bytes) 2019-12-07T19:34:04.747+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72600 onodes shared_blobs 2019-12-07T19:34:04.747+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.747+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72600 prepare 2019-12-07T19:34:04.747+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72600 2019-12-07T19:34:04.747+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72600 io_done 2019-12-07T19:34:04.747+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e84176420 finish 2019-12-07T19:34:04.747+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.747+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.747+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.747+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:04.747+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.747+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1164 < min_flush_size 524288 2019-12-07T19:34:04.747+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1164 < min_flush_size 524288 2019-12-07T19:34:04.747+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.747+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xcb6eb~48c to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.747+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xcb6eb 0xcb6eb~48c to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.747+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.747+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9b6eb 2019-12-07T19:34:04.747+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x6eb 2019-12-07T19:34:04.747+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.747+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb77 and padding block with 0x489 2019-12-07T19:34:04.747+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xcbb77 2019-12-07T19:34:04.747+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.747+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.747+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.747+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.747+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000096092s (0.000000109s flush + 0.000095983s kv commit) 2019-12-07T19:34:04.747+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.747+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.747+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e72600> 2019-12-07T19:34:04.747+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.747+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72600 kv_submitted 2019-12-07T19:34:04.747+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72600 2019-12-07T19:34:04.747+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72600 finishing 2019-12-07T19:34:04.747+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72600 onodes 2019-12-07T19:34:04.747+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72600 done 2019-12-07T19:34:04.747+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.747+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72600 [] 2019-12-07T19:34:04.747+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.747+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.747+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.747+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 27/26 ECSubWriteReply(tid=118, last_complete=21'3, committed=1, applied=1)) v2 -- 0x558e81239800 con 0x558e818d7c00 2019-12-07T19:34:04.747+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.748+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 aio_wait 2019-12-07T19:34:04.748+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e900 2019-12-07T19:34:04.748+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 io_done 2019-12-07T19:34:04.748+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.748+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.748+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.748+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2128 < min_flush_size 524288 2019-12-07T19:34:04.748+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84242c80 had 1 2019-12-07T19:34:04.748+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:04.748+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2151 < min_flush_size 524288 2019-12-07T19:34:04.748+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2151 < min_flush_size 524288 2019-12-07T19:34:04.748+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.748+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xcbb77~867 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.748+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xcbb77 0xcbb77~867 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.748+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.748+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9bb77 2019-12-07T19:34:04.748+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb77 2019-12-07T19:34:04.748+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.748+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x3de and padding block with 0xc22 2019-12-07T19:34:04.749+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xcc3de 2019-12-07T19:34:04.749+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.749+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.749+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.749+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.749+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.001331217s (0.000004827s flush + 0.001326390s kv commit) 2019-12-07T19:34:04.749+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e900> 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.749+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 kv_submitted 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e900 2019-12-07T19:34:04.749+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 finishing 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e900 onodes 0x558e84242c80 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e9588 in 0x558e802196c0) _finish_write discard buffer(0x558e84378510 space 0x558e817e9588 0x0~2000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abd588 in 0x558e802196c0) _finish_write discard buffer(0x558e83d117a0 space 0x558e83abd588 0x0~2000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d029b8 in 0x558e802196c0) _finish_write discard buffer(0x558e84378240 space 0x558e83d029b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0f198 in 0x558e802196c0) _finish_write discard buffer(0x558e83c8e120 space 0x558e83d0f198 0x0~2000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0f908 in 0x558e802196c0) _finish_write discard buffer(0x558e83d10fc0 space 0x558e83d0f908 0x0~2000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d37f98 in 0x558e802196c0) _finish_write discard buffer(0x558e843785a0 space 0x558e83d37f98 0x0~1000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2478 in 0x558e802196c0) _finish_write discard buffer(0x558e84378120 space 0x558e83dd2478 0x0~2000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2d38 in 0x558e802196c0) _finish_write discard buffer(0x558e84378090 space 0x558e83dd2d38 0x0~2000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6f438 in 0x558e802196c0) _finish_write discard buffer(0x558e84378000 space 0x558e83e6f438 0x0~2000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a28d8 in 0x558e802196c0) _finish_write discard buffer(0x558e83d113b0 space 0x558e841a28d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841db908 in 0x558e802196c0) _finish_write discard buffer(0x558e84378480 space 0x558e841db908 0x0~2000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841db9e8 in 0x558e802196c0) _finish_write discard buffer(0x558e843783f0 space 0x558e841db9e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841dbac8 in 0x558e802196c0) _finish_write discard buffer(0x558e84378360 space 0x558e841dbac8 0x0~2000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841dbba8 in 0x558e802196c0) _finish_write discard buffer(0x558e843782d0 space 0x558e841dbba8 0x0~2000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'6 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 0'0 active m=2 mbc={} ps=[2~1]] recovery_committed_to version 21'6 now ondisk 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423ccc8 in 0x558e802196c0) _finish_write discard buffer(0x558e843781b0 space 0x558e8423ccc8 0x0~2000 writing nocache) 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e900 done 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:04.749+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e900 [] 2019-12-07T19:34:04.749+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.749+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.749+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGPushReply(1.3s0 27/26 [PushReplyOp(1:c8ce3553:::smithi13140796-29:head)]) v3 -- 0x558e818a9e00 con 0x558e818fe400 2019-12-07T19:34:04.749+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.750+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 88 ==== MOSDPGRecoveryDelete(1.3s5 e27,26 1:c87708e9:::smithi13140796-17:head,21'7) v2 ==== 123+0+0 (unknown 0 0 0) 0x558e818a9e00 con 0x558e818fe400 2019-12-07T19:34:04.750+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e83edcf20 prio 3 cost 1000 latency 0.000022 epoch 27 MOSDPGRecoveryDelete(1.3s5 e27,26 1:c87708e9:::smithi13140796-17:head,21'7) v2 2019-12-07T19:34:04.750+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDPGRecoveryDelete(1.3s5 e27,26 1:c87708e9:::smithi13140796-17:head,21'7) v2) prio 3 cost 1000 e27) 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDPGRecoveryDelete(1.3s5 e27,26 1:c87708e9:::smithi13140796-17:head,21'7) v2) prio 3 cost 1000 e27) queued 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDPGRecoveryDelete(1.3s5 e27,26 1:c87708e9:::smithi13140796-17:head,21'7) v2) prio 3 cost 1000 e27) pg 0x558e81231000 2019-12-07T19:34:04.750+0000 7fe2c6263700 10 osd.0 27 dequeue_op 0x558e83edcf20 prio 3 cost 1000 latency 0.000054 MOSDPGRecoveryDelete(1.3s5 e27,26 1:c87708e9:::smithi13140796-17:head,21'7) v2 pg pg[1.3s5( v 23'10 lc 21'6 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=2 mbc={} ps=[2~1]] 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'6 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=2 mbc={} ps=[2~1]] handle_recovery_delete 0x558e83edcf20 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'6 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=2 mbc={} ps=[2~1]] remove_missing_object 1:c87708e9:::smithi13140796-17:head 21'7 2019-12-07T19:34:04.750+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e819a0000 seq 5 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) get_onode oid 5#1:c87708e9:::smithi13140796-17:head# key 0x858000000000000001c87708e921736d'ithi13140796-17!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) r -2 v.len 0 2019-12-07T19:34:04.750+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 5#1:c87708e9:::smithi13140796-17:head# 2019-12-07T19:34:04.750+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0000 cost 4212 (1 ios * 4000 + 212 bytes) 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0000 onodes shared_blobs 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0000 allocated 0x[] released 0x[] 2019-12-07T19:34:04.750+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 prepare 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0000 2019-12-07T19:34:04.750+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 io_done 2019-12-07T19:34:04.750+0000 7fe2c6263700 10 osd.0 27 dequeue_op 0x558e83edcf20 finish 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.750+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xcc3de~2e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xcc3de 0xcc3de~2e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9c3de 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x3de 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x40c and padding block with 0xbf4 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xcc40c 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000097179s (0.000000095s flush + 0.000097084s kv commit) 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a0000> 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.750+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 kv_submitted 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0000 2019-12-07T19:34:04.750+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 finishing 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0000 onodes 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0000 done 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:04.750+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0000 [] 2019-12-07T19:34:04.750+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.750+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'6 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=2 mbc={} ps=[2~1]] on_local_recover: 1:c87708e9:::smithi13140796-17:head 2019-12-07T19:34:04.750+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 21'6 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=2 mbc={} ps=[2~1]] got missing 1:c87708e9:::smithi13140796-17:head v 21'7 2019-12-07T19:34:04.750+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 23'9 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 21'6 active m=1 mbc={} ps=[2~1]] last_complete now 23'9 log.complete_to 23'10 2019-12-07T19:34:04.750+0000 7fe2ca26b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.750+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:04.750+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e83c57500 seq 6 2019-12-07T19:34:04.750+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:04.750+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x0000000000000461'.missing/0000000000000001.31EE0179.head.smithi13140796-17..' <- missing/0000000000000001.31EE0179.head.smithi13140796-17.. 2019-12-07T19:34:04.750+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:04.750+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:04.750+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:04.750+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.750+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.750+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:04.750+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c57500 cost 5409 (1 ios * 4000 + 1409 bytes) 2019-12-07T19:34:04.750+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c57500 onodes shared_blobs 2019-12-07T19:34:04.750+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c57500 allocated 0x[] released 0x[] 2019-12-07T19:34:04.750+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 prepare 2019-12-07T19:34:04.750+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c57500 2019-12-07T19:34:04.750+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 io_done 2019-12-07T19:34:04.750+0000 7fe2ca26b700 20 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 23'9 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 21'6 active m=1 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.750+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1185 < min_flush_size 524288 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xcc40c~4b8 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xcc40c 0xcc40c~4b8 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9c40c 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x40c 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x8c4 and padding block with 0x73c 2019-12-07T19:34:04.750+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xcc8c4 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.750+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000128053s (0.000000086s flush + 0.000127967s kv commit) 2019-12-07T19:34:04.750+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c57500> 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.750+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 kv_submitted 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c57500 2019-12-07T19:34:04.750+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 finishing 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c57500 onodes 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57500 done 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:04.750+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c57500 [] 2019-12-07T19:34:04.750+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.750+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.750+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 23'9 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 21'6 active m=1 mbc={} ps=[2~1]] recovery_committed_to version 23'9 now ondisk 2019-12-07T19:34:04.750+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGRecoveryDeleteReply(1.3s0 e27,26 1:c87708e9:::smithi13140796-17:head,21'7) v2 -- 0x558e818a8a00 con 0x558e818fe400 2019-12-07T19:34:04.751+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.751+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.754+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.1 172.21.15.131:0/37078 2 ==== osd_ping(ping e27 up_from 13 ping_stamp 2019-12-07T19:34:04.753767+0000/28.210690433s send_stamp 28.210690433s delta_ub 0.002426586s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8a00 con 0x558e81984c00 2019-12-07T19:34:04.754+0000 7fe2e6d3a700 20 osd.0 27 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.001205621s,0.002426586s]) 2019-12-07T19:34:04.754+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e27 up_from 13 ping_stamp 2019-12-07T19:34:04.753767+0000/28.210690433s send_stamp 28.209484812s delta_ub 18446744073.708345995s) v5 -- 0x558e811f6800 con 0x558e81984c00 2019-12-07T19:34:04.754+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.1 172.21.15.131:0/37078 2 ==== osd_ping(ping e27 up_from 13 ping_stamp 2019-12-07T19:34:04.753767+0000/28.210690433s send_stamp 28.210690433s delta_ub 0.002426586s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81239800 con 0x558e81985000 2019-12-07T19:34:04.754+0000 7fe2e7d3c700 20 osd.0 27 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.001072657s,0.002426586s]) 2019-12-07T19:34:04.754+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e27 up_from 13 ping_stamp 2019-12-07T19:34:04.753767+0000/28.210690433s send_stamp 28.209617776s delta_ub 18446744073.708478959s) v5 -- 0x558e8021f800 con 0x558e81985000 2019-12-07T19:34:04.755+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 386228 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.760+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 89 ==== MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=49, to_read={1:db85c185:::smithi13140796-18:head=0,262144,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e84243400 con 0x558e818fe400 2019-12-07T19:34:04.760+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e83edc000 prio 3 cost 0 latency 0.000026 epoch 27 MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=49, to_read={1:db85c185:::smithi13140796-18:head=0,262144,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.760+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=49, to_read={1:db85c185:::smithi13140796-18:head=0,262144,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=49, to_read={1:db85c185:::smithi13140796-18:head=0,262144,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=49, to_read={1:db85c185:::smithi13140796-18:head=0,262144,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e81928000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 10 osd.0 27 dequeue_op 0x558e83edc000 prio 3 cost 0 latency 0.000067 MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=49, to_read={1:db85c185:::smithi13140796-18:head=0,262144,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.760+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=49, to_read={1:db85c185:::smithi13140796-18:head=0,262144,0}, subchunks={1:db85c185:::smithi13140796-18:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.760+0000 7fe2ca26b700 25 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.760+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 0x0~40000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~40000 size 0x12000 (73728) 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4cb0 blob([0x79000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4c40 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4850 blob([0x7b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f650 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f5e0 blob([0x7d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4d90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4e00 blob([0x7f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4e70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4ee0 blob([0x81000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4f50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4fc0 blob([0x83000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5030 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac50a0 blob([0x85000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5110 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5180 blob([0x87000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac51f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5260 blob([0x89000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac52d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194f5e0 blob([0x7d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4d90 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac4850 blob([0x7b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f650 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac4cb0 blob([0x79000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4c40 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac4e00 blob([0x7f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4e70 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac4ee0 blob([0x81000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4f50 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac4fc0 blob([0x83000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5030 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac50a0 blob([0x85000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5110 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac5180 blob([0x87000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac51f0 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac5260 blob([0x89000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac52d0 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194f5e0 blob([0x7d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4d90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4da8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e83c8ecf0 space 0x558e83ac4da8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac4850 blob([0x7b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f650 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194f668 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e83c8e630 space 0x558e8194f668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac4cb0 blob([0x79000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4c40 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4c58 in 0x558e80219c00) _discard 0x1000~1000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e83d103f0 space 0x558e83ac4c58 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac4e00 blob([0x7f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4e70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4e88 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e83d11e60 space 0x558e83ac4e88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac4ee0 blob([0x81000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4f50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4f68 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e83d11a70 space 0x558e83ac4f68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac4fc0 blob([0x83000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5030 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5048 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e83d11dd0 space 0x558e83ac5048 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac50a0 blob([0x85000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5110 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5128 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e83d11560 space 0x558e83ac5128 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac5180 blob([0x87000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac51f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5208 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843787e0 space 0x558e83ac5208 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac5260 blob([0x89000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac52d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac52e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e84378870 space 0x558e83ac52e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.760+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 0x0~40000 = 73728 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=73728 len=73728 2019-12-07T19:34:04.760+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:04.760+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:04.760+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:04.760+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 73728 2019-12-07T19:34:04.760+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# hinfo_key 2019-12-07T19:34:04.760+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# hinfo_key = 0 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:04.760+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 27/26 ECSubReadReply(tid=49, attrs_read=0)) v2 -- 0x558e8402f900 con 0x558e818fe400 2019-12-07T19:34:04.760+0000 7fe2ca26b700 10 osd.0 27 dequeue_op 0x558e83edc000 finish 2019-12-07T19:34:04.760+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.761+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 90 ==== MOSDPGPush(1.3s5 27/26 [PushOp(1:cd22b62d:::smithi13140796-27:head, version: 23'10, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:cd22b62d:::smithi13140796-27:head@23'10, size: 1376256, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 263206+0+0 (unknown 0 0 0) 0x558e818a9400 con 0x558e818fe400 2019-12-07T19:34:04.761+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e83edc580 prio 127 cost 263144 latency 0.000038 epoch 27 MOSDPGPush(1.3s5 27/26 [PushOp(1:cd22b62d:::smithi13140796-27:head, version: 23'10, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:cd22b62d:::smithi13140796-27:head@23'10, size: 1376256, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.761+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDPGPush(1.3s5 27/26 [PushOp(1:cd22b62d:::smithi13140796-27:head, version: 23'10, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:cd22b62d:::smithi13140796-27:head@23'10, size: 1376256, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e27) 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDPGPush(1.3s5 27/26 [PushOp(1:cd22b62d:::smithi13140796-27:head, version: 23'10, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:cd22b62d:::smithi13140796-27:head@23'10, size: 1376256, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e27) queued 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDPGPush(1.3s5 27/26 [PushOp(1:cd22b62d:::smithi13140796-27:head, version: 23'10, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:cd22b62d:::smithi13140796-27:head@23'10, size: 1376256, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e27) pg 0x558e81231000 2019-12-07T19:34:04.761+0000 7fe2c6263700 10 osd.0 27 dequeue_op 0x558e83edc580 prio 127 cost 263144 latency 0.000102 MOSDPGPush(1.3s5 27/26 [PushOp(1:cd22b62d:::smithi13140796-27:head, version: 23'10, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:cd22b62d:::smithi13140796-27:head@23'10, size: 1376256, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.3s5( v 23'10 lc 23'9 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] 2019-12-07T19:34:04.761+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 23'9 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.3s5 27/26 [PushOp(1:cd22b62d:::smithi13140796-27:head, version: 23'10, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:cd22b62d:::smithi13140796-27:head@23'10, size: 1376256, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 23'9 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] get_temp_recovery_object -3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head 2019-12-07T19:34:04.761+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 23'9 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] handle_recovery_push: Adding oid -3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head in the temp collection 2019-12-07T19:34:04.761+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e819a0900 seq 7 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) get_onode oid 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# key 0x857ffffffffffffffdcd22b6'-!temp_recovering_1.3s5_23'10_26_head!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) r -2 v.len 0 2019-12-07T19:34:04.761+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) get_onode oid 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# key 0x857ffffffffffffffdcd22b6'-!temp_recovering_1.3s5_23'10_26_head!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) r -2 v.len 0 2019-12-07T19:34:04.761+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.3s5_head 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1131 2019-12-07T19:34:04.761+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.3s5_head 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# = 0 2019-12-07T19:34:04.761+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.3s5_head 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# 0x0~40000 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# 0x0~40000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.761+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~40000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0900 32 blobs 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.761+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate 0x40000/1000,40000,0 2019-12-07T19:34:04.762+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x395000~40000/1000,40000,0 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x395000~40000] 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423c540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423c1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423c540 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423c1c0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e8423c540) get_ref 0x0~2000 Blob(0x558e8423c540 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423c1c0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e8423c540) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e8423c540 blob([0x395000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423c1c0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e8423c1d8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423c4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83af9030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423c4d0 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9030 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e8423c4d0) get_ref 0x0~2000 Blob(0x558e8423c4d0 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af9030 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e8423c4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e8423c4d0 blob([0x397000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9030 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83af9048 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a27e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abd9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a27e0 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abd9d0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e841a27e0) get_ref 0x0~2000 Blob(0x558e841a27e0 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abd9d0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e841a27e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e841a27e0 blob([0x399000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abd9d0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83abd9e8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abda40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abdab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abda40 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abdab0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83abda40) get_ref 0x0~2000 Blob(0x558e83abda40 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abdab0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83abda40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83abda40 blob([0x39b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abdab0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83abdac8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abdb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abdb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abdb20 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abdb90 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83abdb20) get_ref 0x0~2000 Blob(0x558e83abdb20 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abdb90 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83abdb20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83abdb20 blob([0x39d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abdb90 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83abdba8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abdc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abdc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abdc00 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abdc70 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83abdc00) get_ref 0x0~2000 Blob(0x558e83abdc00 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abdc70 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83abdc00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e83abdc00 blob([0x39f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abdc70 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83abdc88 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abdce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abdd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abdce0 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abdd50 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83abdce0) get_ref 0x0~2000 Blob(0x558e83abdce0 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abdd50 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83abdce0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e83abdce0 blob([0x3a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abdd50 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83abdd68 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abddc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abde30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abddc0 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abde30 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83abddc0) get_ref 0x0~2000 Blob(0x558e83abddc0 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abde30 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83abddc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 91 ==== MOSDPGPush(1.bs1 27/26 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 74790+0+0 (unknown 0 0 0) 0x558e818a8a00 con 0x558e818fe400 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e83abddc0 blob([0x3a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abde30 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83abde48 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e83edd4a0 prio 127 cost 74728 latency 0.000024 epoch 27 MOSDPGPush(1.bs1 27/26 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abdea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83abdf10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abdea0 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abdf10 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83abdea0) get_ref 0x0~2000 Blob(0x558e83abdea0 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abdf10 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83abdea0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 27/26 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 74728 e27) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83abdea0 blob([0x3a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abdf10 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83abdf28 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83abdf80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed44d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83abdf80 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed44d0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83abdf80) get_ref 0x0~2000 Blob(0x558e83abdf80 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed44d0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83abdf80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83abdf80 blob([0x3a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed44d0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed44e8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 27/26 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 74728 e27) queued 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed4460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed43f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed4460 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed43f0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4460) get_ref 0x0~2000 Blob(0x558e83ed4460 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed43f0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4460) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 27/26 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 74728 e27) pg 0x558e83c78000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83ed4460 blob([0x3a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed43f0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4408 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed4380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed4380 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4310 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4380) get_ref 0x0~2000 Blob(0x558e83ed4380 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4310 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4380) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 osd.0 27 dequeue_op 0x558e83edd4a0 prio 127 cost 74728 latency 0.000067 MOSDPGPush(1.bs1 27/26 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83ed4380 blob([0x3ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4310 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4328 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed42a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.bs1 27/26 [PushOp(1:db85c185:::smithi13140796-18:head, version: 21'2, data_included: [0~73728], data_size: 73728, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:db85c185:::smithi13140796-18:head@21'2, size: 294912, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:294912, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed42a0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4230 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed42a0) get_ref 0x0~2000 Blob(0x558e83ed42a0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4230 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed42a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83ed42a0 blob([0x3ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4230 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4248 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] on_local_recover: 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed41c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed41c0 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4150 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'1 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=4 mbc={} ps=[2~1]] got missing 1:db85c185:::smithi13140796-18:head v 21'2 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed41c0) get_ref 0x0~2000 Blob(0x558e83ed41c0 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4150 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed41c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83ed41c0 blob([0x3af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4150 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4168 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=3 mbc={} ps=[2~1]] last_complete now 21'8 log.complete_to 21'9 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed40e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed40e0 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4000 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed40e0) get_ref 0x0~2000 Blob(0x558e83ed40e0 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4000 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed40e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e83ed40e0 blob([0x3b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4000 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4018 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed4540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed45b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed4540 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed45b0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4540) get_ref 0x0~2000 Blob(0x558e83ed4540 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed45b0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4540) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e8417af00 seq 4 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e83ed4540 blob([0x3b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed45b0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed45c8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed4620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) get_onode oid 1#1:db85c185:::smithi13140796-18:head# key 0x818000000000000001db85c18521736d'ithi13140796-18!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed4620 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4690 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4620) get_ref 0x0~2000 Blob(0x558e83ed4620 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4690 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4620) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e83ed4620 blob([0x3b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4690 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) r -2 v.len 0 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed46a8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) get_onode oid 1#1:db85c185:::smithi13140796-18:head# key 0x818000000000000001db85c18521736d'ithi13140796-18!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed4700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) r -2 v.len 0 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed4700 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4770 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1132 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4700) get_ref 0x0~2000 Blob(0x558e83ed4700 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4770 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4700) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# = 0 2019-12-07T19:34:04.762+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x0~12000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e83ed4700 blob([0x3b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4770 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 1#1:db85c185:::smithi13140796-18:head# 0x0~12000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4788 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~12000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed47e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed47e0 blob([0x3b9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4850 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed47e0) get_ref 0x0~2000 Blob(0x558e83ed47e0 blob([0x3b9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4850 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8417af00 9 blobs 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed47e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e83ed47e0 blob([0x3b9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4850 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4868 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed48c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed48c0 blob([0x3bb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4930 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed48c0) get_ref 0x0~2000 Blob(0x558e83ed48c0 blob([0x3bb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4930 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed48c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e83ed48c0 blob([0x3bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4930 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4948 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed49a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed49a0 blob([0x3bd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4a10 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed49a0) get_ref 0x0~2000 Blob(0x558e83ed49a0 blob([0x3bd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4a10 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed49a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e83ed49a0 blob([0x3bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4a10 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4a28 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 fbmap_alloc 0x558e80233300 allocate 0x12000/1000,12000,0 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed4a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x3d5000~12000/1000,12000,0 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x3d5000~12000] 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed4a80 blob([0x3bf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4af0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83edeee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f943f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4a80) get_ref 0x0~2000 Blob(0x558e83ed4a80 blob([0x3bf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4af0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83edeee0 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f943f0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4a80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e83edeee0) get_ref 0x0~2000 Blob(0x558e83edeee0 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f943f0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e83edeee0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558e83ed4a80 blob([0x3bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4af0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4b08 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83edeee0 blob([0x3d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f943f0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83f94408 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed4b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0f9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e840141c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed4b60 blob([0x3c1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4bd0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0f9d0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840141c0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4b60) get_ref 0x0~2000 Blob(0x558e83ed4b60 blob([0x3c1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4bd0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d0f9d0) get_ref 0x0~2000 Blob(0x558e83d0f9d0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840141c0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4b60) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d0f9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83d0f9d0 blob([0x3d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840141c0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558e83ed4b60 blob([0x3c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4bd0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e840141d8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4be8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d031f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d031f0 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03180 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed4c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a2690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d031f0) get_ref 0x0~2000 Blob(0x558e83d031f0 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03180 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d031f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed4c40 blob([0x3c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a2690 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83d031f0 blob([0x3d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03180 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83d03198 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4c40) get_ref 0x0~2000 Blob(0x558e83ed4c40 blob([0x3c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a2690 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4c40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a2770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6d7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a2770 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6d7a0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558e83ed4c40 blob([0x3c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2690 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841a26a8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e841a2770) get_ref 0x0~2000 Blob(0x558e841a2770 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6d7a0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e841a2770) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e841a2770 blob([0x3db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6d7a0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6d7b8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a2620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a25b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6d730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6d6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6d730 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6d6c0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a2620 blob([0x3c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a25b0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6d730) get_ref 0x0~2000 Blob(0x558e83e6d730 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6d6c0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6d730) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e841a2620) get_ref 0x0~2000 Blob(0x558e841a2620 blob([0x3c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a25b0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83e6d730 blob([0x3dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6d6c0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e841a2620) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6d6d8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558e841a2620 blob([0x3c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a25b0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841a25c8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7a000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7a000 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a070 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e81a7a000) get_ref 0x0~2000 Blob(0x558e81a7a000 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a070 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e81a7a000) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e81a7a000 blob([0x3df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7a070 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed4cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e81a7a088 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed4cb0 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4d20 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7a0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7a0e0 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a150 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4cb0) get_ref 0x0~2000 Blob(0x558e83ed4cb0 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4d20 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e81a7a0e0) get_ref 0x0~2000 Blob(0x558e81a7a0e0 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a150 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e81a7a0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e81a7a0e0 blob([0x3e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7a150 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e81a7a168 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558e83ed4cb0 blob([0x3c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4d20 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4d38 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7a1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7a1c0 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a230 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed4d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e81a7a1c0) get_ref 0x0~2000 Blob(0x558e81a7a1c0 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a230 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e81a7a1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e81a7a1c0 blob([0x3e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7a230 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed4d90 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4e00 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e81a7a248 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4d90) get_ref 0x0~2000 Blob(0x558e83ed4d90 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4e00 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7a2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4d90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7a2a0 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a310 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e81a7a2a0) get_ref 0x0~2000 Blob(0x558e81a7a2a0 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a310 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558e83ed4d90 blob([0x3c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4e00 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.blob(0x558e81a7a2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4e18 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e81a7a2a0 blob([0x3e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7a310 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e81a7a328 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed4e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x12000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed4e70 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4ee0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4e70) get_ref 0x0~2000 Blob(0x558e83ed4e70 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4ee0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e84243508) dirty_range mark inline shard dirty 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4e70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 0x0~12000 = 0 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558e83ed4e70 blob([0x3cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4ee0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4ef8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 4 keys 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed4f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs1_head 1#1:db85c185:::smithi13140796-18:head# 4 keys = 0 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed4f50 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4fc0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4f50) get_ref 0x0~2000 Blob(0x558e83ed4f50 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed4fc0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed4f50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x0000000000000460'.missing/0000000000000001.BD1A381A.head.smithi13140796-18..' <- missing/0000000000000001.BD1A381A.head.smithi13140796-18.. 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558e83ed4f50 blob([0x3cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed4fc0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed4fd8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed5030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed50a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed5030 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed50a0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed5030) get_ref 0x0~2000 Blob(0x558e83ed5030 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed50a0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed5030) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417af00 cost 115965 (10 ios * 4000 + 75965 bytes) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417af00 onodes 0x558e84243400 shared_blobs 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558e83ed5030 blob([0x3cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed50a0 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e84243508) update 1#1:db85c185:::smithi13140796-18:head# 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed50b8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e84243508) update inline shard 174 bytes from 9 extents 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed5110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed5180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:db85c185:::smithi13140796-18:head# is 665 (485 bytes onode + 2 bytes spanning blobs + 178 bytes inline extents) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed5110 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed5180 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417af00 allocated 0x[3d5000~12000] released 0x[] 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 freelist allocate 0x3d5000~12000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 freelist _xor first_key 0x380000 last_key 0x380000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed5110) get_ref 0x0~2000 Blob(0x558e83ed5110 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed5180 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed5110) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558e83ed5110 blob([0x3d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed5180 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed5198 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed51f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ed5260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 prepare 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed51f0 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed5260 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8417af00 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed51f0) get_ref 0x0~2000 Blob(0x558e83ed51f0 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ed5260 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.blob(0x558e83ed51f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558e83ed51f0 blob([0x3d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ed5260 sbid 0x0)) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ed5278 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x40000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84243008) dirty_range mark inline shard dirty 2019-12-07T19:34:04.762+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.3s5_head 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# 0x0~40000 = 0 2019-12-07T19:34:04.762+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.3s5_head 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# 4 keys 2019-12-07T19:34:04.762+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.3s5_head 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# 4 keys = 0 2019-12-07T19:34:04.762+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0900 cost 395060 (33 ios * 4000 + 263060 bytes) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0900 onodes 0x558e84242f00 shared_blobs 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84243008) update 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84243008) update inline shard 611 bytes from 32 extents 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# is 1102 (485 bytes onode + 2 bytes spanning blobs + 615 bytes inline extents) 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0900 allocated 0x[395000~40000] released 0x[] 2019-12-07T19:34:04.762+0000 7fe2c6263700 10 freelist allocate 0x395000~40000 2019-12-07T19:34:04.762+0000 7fe2c6263700 20 freelist _xor first_key 0x380000 last_key 0x380000 2019-12-07T19:34:04.762+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 prepare 2019-12-07T19:34:04.762+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e819a0900 2019-12-07T19:34:04.762+0000 7fe2ca26b700 10 osd.0 27 dequeue_op 0x558e83edd4a0 finish 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=3 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.762+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 aio_wait 2019-12-07T19:34:04.762+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417af00 2019-12-07T19:34:04.762+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 io_done 2019-12-07T19:34:04.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1988 < min_flush_size 524288 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84243400 had 1 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2011 < min_flush_size 524288 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2011 < min_flush_size 524288 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xcc8c4~7db to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xcc8c4 0xcc8c4~7db to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9c8c4 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x8c4 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x9f and padding block with 0xf61 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xcd09f 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000202193s (0.000050163s flush + 0.000152030s kv commit) 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417af00> 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 kv_submitted 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417af00 2019-12-07T19:34:04.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 finishing 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417af00 onodes 0x558e84243400 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7a088 in 0x558e80219c00) _finish_write discard buffer(0x558e84378c60 space 0x558e81a7a088 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7a168 in 0x558e80219c00) _finish_write discard buffer(0x558e84379e60 space 0x558e81a7a168 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7a248 in 0x558e80219c00) _finish_write discard buffer(0x558e84379ef0 space 0x558e81a7a248 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7a328 in 0x558e80219c00) _finish_write discard buffer(0x558e843ba000 space 0x558e81a7a328 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d03198 in 0x558e80219c00) _finish_write discard buffer(0x558e84378ab0 space 0x558e83d03198 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6d6d8 in 0x558e80219c00) _finish_write discard buffer(0x558e84378bd0 space 0x558e83e6d6d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6d7b8 in 0x558e80219c00) _finish_write discard buffer(0x558e84378b40 space 0x558e83e6d7b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f94408 in 0x558e80219c00) _finish_write discard buffer(0x558e84378990 space 0x558e83f94408 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e840141d8 in 0x558e80219c00) _finish_write discard buffer(0x558e84378a20 space 0x558e840141d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417af00 done 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:04.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417af00 [] 2019-12-07T19:34:04.763+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.763+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 0'0 active m=3 mbc={} ps=[2~1]] recovery_committed_to version 21'8 now ondisk 2019-12-07T19:34:04.763+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGPushReply(1.bs0 27/26 [PushReplyOp(1:db85c185:::smithi13140796-18:head)]) v3 -- 0x558e84065600 con 0x558e818fe400 2019-12-07T19:34:04.763+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.763+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.763+0000 7fe2c6263700 10 osd.0 27 dequeue_op 0x558e83edc580 finish 2019-12-07T19:34:04.763+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.763+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 aio_wait 2019-12-07T19:34:04.763+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0900 2019-12-07T19:34:04.763+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 io_done 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1281 < min_flush_size 524288 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84242f00 had 1 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1304 < min_flush_size 524288 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1304 < min_flush_size 524288 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xcd09f~518 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xcd09f 0xcd09f~518 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9d09f 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x9f 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x5b7 and padding block with 0xa49 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xcd5b7 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.763+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000096176s (0.000029729s flush + 0.000066447s kv commit) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a0900> 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 kv_submitted 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0900 2019-12-07T19:34:04.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 finishing 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0900 onodes 0x558e84242f00 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abd9e8 in 0x558e802196c0) _finish_write discard buffer(0x558e84378cf0 space 0x558e83abd9e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abdac8 in 0x558e802196c0) _finish_write discard buffer(0x558e84378d80 space 0x558e83abdac8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abdba8 in 0x558e802196c0) _finish_write discard buffer(0x558e84378e10 space 0x558e83abdba8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abdc88 in 0x558e802196c0) _finish_write discard buffer(0x558e84378ea0 space 0x558e83abdc88 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abdd68 in 0x558e802196c0) _finish_write discard buffer(0x558e84378f30 space 0x558e83abdd68 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abde48 in 0x558e802196c0) _finish_write discard buffer(0x558e84378fc0 space 0x558e83abde48 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83abdf28 in 0x558e802196c0) _finish_write discard buffer(0x558e84379050 space 0x558e83abdf28 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGPushReply(1.3s0 27/26 [PushReplyOp(1:cd22b62d:::smithi13140796-27:head)]) v3 -- 0x558e818a9e00 con 0x558e818fe400 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83af9048 in 0x558e802196c0) _finish_write discard buffer(0x558e84378750 space 0x558e83af9048 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4018 in 0x558e802196c0) _finish_write discard buffer(0x558e843793b0 space 0x558e83ed4018 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4168 in 0x558e802196c0) _finish_write discard buffer(0x558e84379320 space 0x558e83ed4168 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4248 in 0x558e802196c0) _finish_write discard buffer(0x558e84379290 space 0x558e83ed4248 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4328 in 0x558e802196c0) _finish_write discard buffer(0x558e84379200 space 0x558e83ed4328 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4408 in 0x558e802196c0) _finish_write discard buffer(0x558e84379170 space 0x558e83ed4408 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed44e8 in 0x558e802196c0) _finish_write discard buffer(0x558e843790e0 space 0x558e83ed44e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed45c8 in 0x558e802196c0) _finish_write discard buffer(0x558e84379440 space 0x558e83ed45c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed46a8 in 0x558e802196c0) _finish_write discard buffer(0x558e843794d0 space 0x558e83ed46a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4788 in 0x558e802196c0) _finish_write discard buffer(0x558e84379560 space 0x558e83ed4788 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4868 in 0x558e802196c0) _finish_write discard buffer(0x558e843795f0 space 0x558e83ed4868 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4948 in 0x558e802196c0) _finish_write discard buffer(0x558e84379680 space 0x558e83ed4948 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4a28 in 0x558e802196c0) _finish_write discard buffer(0x558e84379710 space 0x558e83ed4a28 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4b08 in 0x558e802196c0) _finish_write discard buffer(0x558e843797a0 space 0x558e83ed4b08 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4be8 in 0x558e802196c0) _finish_write discard buffer(0x558e84379830 space 0x558e83ed4be8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4d38 in 0x558e802196c0) _finish_write discard buffer(0x558e843799e0 space 0x558e83ed4d38 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4e18 in 0x558e802196c0) _finish_write discard buffer(0x558e84379a70 space 0x558e83ed4e18 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4ef8 in 0x558e802196c0) _finish_write discard buffer(0x558e84379b00 space 0x558e83ed4ef8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed4fd8 in 0x558e802196c0) _finish_write discard buffer(0x558e84379b90 space 0x558e83ed4fd8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed50b8 in 0x558e802196c0) _finish_write discard buffer(0x558e84379c20 space 0x558e83ed50b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed5198 in 0x558e802196c0) _finish_write discard buffer(0x558e84379cb0 space 0x558e83ed5198 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ed5278 in 0x558e802196c0) _finish_write discard buffer(0x558e84379d40 space 0x558e83ed5278 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a25c8 in 0x558e802196c0) _finish_write discard buffer(0x558e84379950 space 0x558e841a25c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a26a8 in 0x558e802196c0) _finish_write discard buffer(0x558e843798c0 space 0x558e841a26a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423c1d8 in 0x558e802196c0) _finish_write discard buffer(0x558e83c8e090 space 0x558e8423c1d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0900 done 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:04.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0900 [] 2019-12-07T19:34:04.763+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.764+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 92 ==== MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=51, to_read={1:dce63cc3:::smithi13140796-50:head=0,262144,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8402f900 con 0x558e818fe400 2019-12-07T19:34:04.764+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e81943600 prio 3 cost 0 latency 0.000023 epoch 27 MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=51, to_read={1:dce63cc3:::smithi13140796-50:head=0,262144,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.764+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=51, to_read={1:dce63cc3:::smithi13140796-50:head=0,262144,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=51, to_read={1:dce63cc3:::smithi13140796-50:head=0,262144,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=51, to_read={1:dce63cc3:::smithi13140796-50:head=0,262144,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e81928000 2019-12-07T19:34:04.764+0000 7fe2c6263700 10 osd.0 27 dequeue_op 0x558e81943600 prio 3 cost 0 latency 0.000072 MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=51, to_read={1:dce63cc3:::smithi13140796-50:head=0,262144,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.764+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=51, to_read={1:dce63cc3:::smithi13140796-50:head=0,262144,0}, subchunks={1:dce63cc3:::smithi13140796-50:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.764+0000 7fe2c6263700 25 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.764+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 0x0~40000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~40000 size 0x33000 (208896) 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d02620 blob([0x2bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48e00 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37030 blob([0x2c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4bd0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0fb20 blob([0x2c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fb90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0fc00 blob([0x2c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fc70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0fce0 blob([0x2c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015490 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84015500 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015570 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840155e0 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015650 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840156c0 blob([0x2cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015730 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840157a0 blob([0x2cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015810 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84015880 blob([0x2d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840158f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84015960 blob([0x2d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840159d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84015a40 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015ab0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84015b20 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015b90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84015c00 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015c70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84030a10 blob([0x2f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fa40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6c310 blob([0x2f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c380 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6c460 blob([0x2f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c4d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6c540 blob([0x2f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c5b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6c620 blob([0x2fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c690 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6c700 blob([0x2fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6df80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6df10 blob([0x2fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dea0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6de30 blob([0x300000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6ddc0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6dd50 blob([0x302000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dce0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6dc70 blob([0x304000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dc00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6db90 blob([0x306000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6db20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6dab0 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83e6da40 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d02620 blob([0x2bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48e00 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0fb20 blob([0x2c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fb90 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0fc00 blob([0x2c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fc70 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0fce0 blob([0x2c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015490 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d37030 blob([0x2c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4bd0 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6c310 blob([0x2f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c380 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6c460 blob([0x2f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c4d0 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6c540 blob([0x2f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c5b0 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6c620 blob([0x2fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c690 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6c700 blob([0x2fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6df80 sbid 0x0)) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6dab0 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83e6da40 sbid 0x0)) need {<0x0, 0x1000> : [0x32000:0~1000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x32000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6db90 blob([0x306000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6db20 sbid 0x0)) need {<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x30000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6dc70 blob([0x304000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dc00 sbid 0x0)) need {<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2e000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6dd50 blob([0x302000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dce0 sbid 0x0)) need {<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2c000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6de30 blob([0x300000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6ddc0 sbid 0x0)) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6df10 blob([0x2fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dea0 sbid 0x0)) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84015500 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015570 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e840155e0 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015650 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e840156c0 blob([0x2cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015730 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e840157a0 blob([0x2cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015810 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84015880 blob([0x2d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840158f0 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84015960 blob([0x2d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840159d0 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84015a40 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015ab0 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84015b20 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015b90 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84015c00 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015c70 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84030a10 blob([0x2f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fa40 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d02620 blob([0x2bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48e00 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83f48e18 in 0x558e80219c00) _discard 0x1000~1000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843ba6c0 space 0x558e83f48e18 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0fb20 blob([0x2c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fb90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0fba8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843ba750 space 0x558e83d0fba8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0fc00 blob([0x2c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fc70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0fc88 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843ba7e0 space 0x558e83d0fc88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0fce0 blob([0x2c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015490 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e840154a8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843ba870 space 0x558e840154a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d37030 blob([0x2c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4bd0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83ac4be8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843ba900 space 0x558e83ac4be8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6c310 blob([0x2f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c380 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6c398 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843ba990 space 0x558e83e6c398 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6c460 blob([0x2f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c4d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6c4e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843baa20 space 0x558e83e6c4e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6c540 blob([0x2f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c5b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:04.764+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6c5c8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843baab0 space 0x558e83e6c5c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6c620 blob([0x2fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c690 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6c6a8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bab40 space 0x558e83e6c6a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6c700 blob([0x2fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6df80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6df98 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843babd0 space 0x558e83e6df98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6dab0 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83e6da40 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x32000:0~1000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6da58 in 0x558e80219c00) _discard 0x0~1000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bac60 space 0x558e83e6da58 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6db90 blob([0x306000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6db20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6db38 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bacf0 space 0x558e83e6db38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6dc70 blob([0x304000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dc00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6dc18 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bad80 space 0x558e83e6dc18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6dd50 blob([0x302000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dce0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6dcf8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bae10 space 0x558e83e6dcf8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6de30 blob([0x300000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6ddc0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6ddd8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843baea0 space 0x558e83e6ddd8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6df10 blob([0x2fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dea0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83e6deb8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843baf30 space 0x558e83e6deb8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84015500 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015570 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015588 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bafc0 space 0x558e84015588 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e840155e0 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015650 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015668 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bb050 space 0x558e84015668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e840156c0 blob([0x2cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015730 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015748 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bb0e0 space 0x558e84015748 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e840157a0 blob([0x2cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015810 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015828 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bb170 space 0x558e84015828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84015880 blob([0x2d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840158f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015908 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bb200 space 0x558e84015908 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84015960 blob([0x2d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840159d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e840159e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bb290 space 0x558e840159e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84015a40 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015ab0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015ac8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bb320 space 0x558e84015ac8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84015b20 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015b90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015ba8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bb3b0 space 0x558e84015ba8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84015c00 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015c70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84015c88 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bb440 space 0x558e84015c88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84030a10 blob([0x2f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fa40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0fa58 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bb4d0 space 0x558e83d0fa58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 0x0~40000 = 208896 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=208896 len=208896 2019-12-07T19:34:04.765+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:04.765+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:04.765+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:04.765+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 208896 2019-12-07T19:34:04.765+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# hinfo_key 2019-12-07T19:34:04.765+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# hinfo_key = 0 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:04.765+0000 7fe2c6263700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 27/26 ECSubReadReply(tid=51, attrs_read=0)) v2 -- 0x558e84243180 con 0x558e818fe400 2019-12-07T19:34:04.765+0000 7fe2c6263700 10 osd.0 27 dequeue_op 0x558e81943600 finish 2019-12-07T19:34:04.765+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.768+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 93 ==== MOSDPGPush(1.bs1 27/26 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 209958+0+0 (unknown 0 0 0) 0x558e818eea00 con 0x558e818fe400 2019-12-07T19:34:04.768+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e81943b80 prio 127 cost 209896 latency 0.000027 epoch 27 MOSDPGPush(1.bs1 27/26 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.768+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 27/26 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 209896 e27) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 27/26 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 209896 e27) queued 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 27/26 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 209896 e27) pg 0x558e83c78000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 osd.0 27 dequeue_op 0x558e81943b80 prio 127 cost 209896 latency 0.000075 MOSDPGPush(1.bs1 27/26 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.bs1( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=3 mbc={} ps=[2~1]] 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=3 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.bs1 27/26 [PushOp(1:dce63cc3:::smithi13140796-50:head, version: 21'9, data_included: [0~208896], data_size: 208896, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:dce63cc3:::smithi13140796-50:head@21'9, size: 835584, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:835584, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=3 mbc={} ps=[2~1]] on_local_recover: 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'8 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=3 mbc={} ps=[2~1]] got missing 1:dce63cc3:::smithi13140796-50:head v 21'9 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'8 active m=2 mbc={} ps=[2~1]] last_complete now 21'9 log.complete_to 21'10 2019-12-07T19:34:04.768+0000 7fe2ca26b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.768+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 94 ==== MOSDPGPush(1.3s5 27/26 [PushOp(1:cd22b62d:::smithi13140796-27:head, version: 23'10, data_included: [262144~81920], data_size: 81920, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:cd22b62d:::smithi13140796-27:head@23'10, size: 1376256, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1376256, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 82520+0+0 (unknown 0 0 0) 0x558e84065600 con 0x558e818fe400 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e8417a600 seq 5 2019-12-07T19:34:04.768+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e81942f20 prio 127 cost 82920 latency 0.000030 epoch 27 MOSDPGPush(1.3s5 27/26 [PushOp(1:cd22b62d:::smithi13140796-27:head, version: 23'10, data_included: [262144~81920], data_size: 81920, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:cd22b62d:::smithi13140796-27:head@23'10, size: 1376256, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1376256, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) get_onode oid 1#1:dce63cc3:::smithi13140796-50:head# key 0x818000000000000001dce63cc321736d'ithi13140796-50!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.768+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDPGPush(1.3s5 27/26 [PushOp(1:cd22b62d:::smithi13140796-27:head, version: 23'10, data_included: [262144~81920], data_size: 81920, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:cd22b62d:::smithi13140796-27:head@23'10, size: 1376256, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1376256, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 82920 e27) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) r -2 v.len 0 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) get_onode oid 1#1:dce63cc3:::smithi13140796-50:head# key 0x818000000000000001dce63cc321736d'ithi13140796-50!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) r -2 v.len 0 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDPGPush(1.3s5 27/26 [PushOp(1:cd22b62d:::smithi13140796-27:head, version: 23'10, data_included: [262144~81920], data_size: 81920, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:cd22b62d:::smithi13140796-27:head@23'10, size: 1376256, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1376256, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 82920 e27) queued 2019-12-07T19:34:04.768+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1133 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# = 0 2019-12-07T19:34:04.768+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 0x0~33000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 1#1:dce63cc3:::smithi13140796-50:head# 0x0~33000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~33000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDPGPush(1.3s5 27/26 [PushOp(1:cd22b62d:::smithi13140796-27:head, version: 23'10, data_included: [262144~81920], data_size: 81920, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:cd22b62d:::smithi13140796-27:head@23'10, size: 1376256, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1376256, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 82920 e27) pg 0x558e81231000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8417a600 26 blobs 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 osd.0 27 dequeue_op 0x558e81942f20 prio 127 cost 82920 latency 0.000089 MOSDPGPush(1.3s5 27/26 [PushOp(1:cd22b62d:::smithi13140796-27:head, version: 23'10, data_included: [262144~81920], data_size: 81920, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:cd22b62d:::smithi13140796-27:head@23'10, size: 1376256, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1376256, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.3s5( v 23'10 lc 23'9 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 23'9 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.3s5 27/26 [PushOp(1:cd22b62d:::smithi13140796-27:head, version: 23'10, data_included: [262144~81920], data_size: 81920, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:cd22b62d:::smithi13140796-27:head@23'10, size: 1376256, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1376256, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 23'9 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] get_temp_recovery_object -3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 23'9 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] handle_recovery_push: Removing oid -3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head from the temp collection 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 23'9 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] on_local_recover: 1:cd22b62d:::smithi13140796-27:head 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 lc 23'9 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active m=1 mbc={} ps=[2~1]] got missing 1:cd22b62d:::smithi13140796-27:head v 23'10 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 23'9 active mbc={} ps=[2~1]] last_complete now 23'10 log.complete_to at end 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 fbmap_alloc 0x558e80233300 allocate 0x33000/1000,33000,0 2019-12-07T19:34:04.768+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x3e7000~33000/1000,33000,0 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x3e7000~33000] 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e840147e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84014bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e840147e0 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84014bd0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e840147e0) get_ref 0x0~2000 Blob(0x558e840147e0 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84014bd0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e840147e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e840147e0 blob([0x3e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84014bd0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84014be8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84014c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edf2d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84014c40 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf2d0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e819a1b00 seq 8 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84014c40) get_ref 0x0~2000 Blob(0x558e84014c40 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf2d0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84014c40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e84014c40 blob([0x3e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edf2d0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83edf2e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.3s5_head 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# 0x40000~14000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7a770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84462000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7a770 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462000 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# 0x40000~14000 - have 0x40000 (262144) bytes fadvise_flags 0x0 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e81a7a770) get_ref 0x0~2000 Blob(0x558e81a7a770 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462000 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e81a7a770) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e81a7a770 blob([0x3eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84462000 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84462018 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x40000~14000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e844620e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462070 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e844620e0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462070) get_ref 0x0~2000 Blob(0x558e84462070 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e844620e0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462070) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e84462070 blob([0x3ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e844620e0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e844620f8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e844621c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a1b00 10 blobs 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462150 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e844621c0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462150) get_ref 0x0~2000 Blob(0x558e84462150 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e844621c0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462150) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e84462150 blob([0x3ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e844621c0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e844621d8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e844622a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462230 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e844622a0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462230) get_ref 0x0~2000 Blob(0x558e84462230 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e844622a0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462230) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e84462230 blob([0x3f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e844622a0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e844622b8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84462380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462310 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462380 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462310) get_ref 0x0~2000 Blob(0x558e84462310 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462380 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462310) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e84462310 blob([0x3f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84462380 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84462398 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e844623f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84462460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e844623f0 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462460 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e844623f0) get_ref 0x0~2000 Blob(0x558e844623f0 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462460 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e844623f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e844623f0 blob([0x3f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84462460 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84462478 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e844624d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84462540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e844624d0 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462540 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e844624d0) get_ref 0x0~2000 Blob(0x558e844624d0 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462540 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e844624d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e844624d0 blob([0x3f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84462540 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84462558 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e844625b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84462620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e844625b0 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462620 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e844625b0) get_ref 0x0~2000 Blob(0x558e844625b0 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462620 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e844625b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e844625b0 blob([0x3f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84462620 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84462638 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84462700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462690 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462700 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462690) get_ref 0x0~2000 Blob(0x558e84462690 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462700 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462690) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e84462690 blob([0x3fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84462700 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84462718 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e844627e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462770 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e844627e0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462770) get_ref 0x0~2000 Blob(0x558e84462770 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e844627e0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462770) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e84462770 blob([0x3fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e844627e0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e844627f8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate 0x14000/1000,14000,0 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e844628c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x41a000~14000/1000,14000,0 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462850 blob([0x3ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e844628c0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x41a000~14000] 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462850) get_ref 0x0~2000 Blob(0x558e84462850 blob([0x3ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e844628c0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462850) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e84462850 blob([0x3ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e844628c0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e840149a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e844628d8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e840149a0 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a8c0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e844629a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462930 blob([0x401000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e844629a0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462930) get_ref 0x0~2000 Blob(0x558e84462930 blob([0x401000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e844629a0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e840149a0) get_ref 0x0~2000 Blob(0x558e840149a0 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a8c0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462930) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e840149a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e84462930 blob([0x401000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e844629a0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e844629b8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84462a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558e840149a0 blob([0x41a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7a8c0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462a10 blob([0x403000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462a80 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a7a8d8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462a10) get_ref 0x0~2000 Blob(0x558e84462a10 blob([0x403000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462a80 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462a10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e84462a10 blob([0x403000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84462a80 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84462a98 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84462b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462af0 blob([0x405000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462b60 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84014a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423c460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462af0) get_ref 0x0~2000 Blob(0x558e84462af0 blob([0x405000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462b60 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462af0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84014a80 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423c460 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e84462af0 blob([0x405000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84462b60 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84462b78 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e84014a80) get_ref 0x0~2000 Blob(0x558e84014a80 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423c460 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84462c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e84014a80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462bd0 blob([0x407000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462c40 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462bd0) get_ref 0x0~2000 Blob(0x558e84462bd0 blob([0x407000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462c40 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558e84014a80 blob([0x41c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423c460 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e84462bd0 blob([0x407000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84462c40 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e8423c478 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84462c58 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84462d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462cb0 blob([0x409000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462d20 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7ae00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7ae70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462cb0) get_ref 0x0~2000 Blob(0x558e84462cb0 blob([0x409000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462d20 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7ae00 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7ae70 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e84462cb0 blob([0x409000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84462d20 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84462d38 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7ae00) get_ref 0x0~2000 Blob(0x558e81a7ae00 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7ae70 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84462e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7ae00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462d90 blob([0x40b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462e00 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558e81a7ae00 blob([0x41e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7ae70 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462d90) get_ref 0x0~2000 Blob(0x558e84462d90 blob([0x40b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462e00 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462d90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a7ae88 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e84462d90 blob([0x40b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84462e00 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84462e18 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84462ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462e70 blob([0x40d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462ee0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7aee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7af50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462e70) get_ref 0x0~2000 Blob(0x558e84462e70 blob([0x40d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462ee0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462e70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7aee0 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7af50 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e84462e70 blob([0x40d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84462ee0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84462ef8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7aee0) get_ref 0x0~2000 Blob(0x558e81a7aee0 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7af50 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7aee0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84462f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84462fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84462f50 blob([0x40f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462fc0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558e81a7aee0 blob([0x420000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7af50 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a7af68 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462f50) get_ref 0x0~2000 Blob(0x558e84462f50 blob([0x40f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84462fc0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84462f50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e84462f50 blob([0x40f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84462fc0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84462fd8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7afc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7afc0 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b030 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84463030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e844630a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84463030 blob([0x411000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e844630a0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7afc0) get_ref 0x0~2000 Blob(0x558e81a7afc0 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b030 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7afc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84463030) get_ref 0x0~2000 Blob(0x558e84463030 blob([0x411000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e844630a0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84463030) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558e84463030 blob([0x411000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e844630a0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558e81a7afc0 blob([0x422000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7b030 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e844630b8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a7b048 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84463110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84463180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84463110 blob([0x413000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84463180 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7b0a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84463110) get_ref 0x0~2000 Blob(0x558e84463110 blob([0x413000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84463180 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e84463110) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558e84463110 blob([0x413000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84463180 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7b0a0 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b110 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84463198 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7b0a0) get_ref 0x0~2000 Blob(0x558e81a7b0a0 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b110 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e844631f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84463260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7b0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e844631f0 blob([0x415000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84463260 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e844631f0) get_ref 0x0~2000 Blob(0x558e844631f0 blob([0x415000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84463260 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558e81a7b0a0 blob([0x424000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7b110 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e844631f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a7b128 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558e844631f0 blob([0x415000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84463260 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84463278 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7b180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b1f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7b180 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b1f0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e844632d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84463340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e844632d0 blob([0x417000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84463340 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7b180) get_ref 0x0~2000 Blob(0x558e81a7b180 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b1f0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e844632d0) get_ref 0x0~2000 Blob(0x558e844632d0 blob([0x417000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84463340 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7b180) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e844632d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558e844632d0 blob([0x417000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84463340 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84463358 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4c000~2000: 0x0~2000 Blob(0x558e81a7b180 blob([0x426000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7b1f0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a7b208 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e844633b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84463420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e844633b0 blob([0x419000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84463420 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e844633b0) get_ref 0x0~1000 Blob(0x558e844633b0 blob([0x419000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84463420 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.blob(0x558e844633b0) get_ref init 0x1000, 1000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7a930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x32000~1000: 0x0~1000 Blob(0x558e844633b0 blob([0x419000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e84463420 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84463438 in 0x558e80219c00) _discard 0x0~1000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7a930 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a9a0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x33000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7a930) get_ref 0x0~2000 Blob(0x558e81a7a930 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7a9a0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7a930) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e84243a08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4e000~2000: 0x0~2000 Blob(0x558e81a7a930 blob([0x428000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7a9a0 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a7a9b8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 0x0~33000 = 0 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7aa10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7aa80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 4 keys 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7aa10 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7aa80 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7aa10) get_ref 0x0~2000 Blob(0x558e81a7aa10 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7aa80 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs1_head 1#1:dce63cc3:::smithi13140796-50:head# 4 keys = 0 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7aa10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x50000~2000: 0x0~2000 Blob(0x558e81a7aa10 blob([0x42a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7aa80 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a7aa98 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x0000000000000460'.missing/0000000000000001.B376C33C.head.smithi13140796-50..' <- missing/0000000000000001.B376C33C.head.smithi13140796-50.. 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7aaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7ad90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7aaf0 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7ad90 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7aaf0) get_ref 0x0~2000 Blob(0x558e81a7aaf0 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7ad90 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7aaf0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x52000~2000: 0x0~2000 Blob(0x558e81a7aaf0 blob([0x42c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7ad90 sbid 0x0)) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a7ada8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417a600 cost 319133 (27 ios * 4000 + 211133 bytes) 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417a600 onodes 0x558e84243900 shared_blobs 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e84243a08) update 1#1:dce63cc3:::smithi13140796-50:head# 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x54000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e84243a08) update inline shard 494 bytes from 26 extents 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84243008) dirty_range mark inline shard dirty 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:dce63cc3:::smithi13140796-50:head# is 985 (485 bytes onode + 2 bytes spanning blobs + 498 bytes inline extents) 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.3s5_head 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# 0x40000~14000 = 0 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417a600 allocated 0x[3e7000~33000] released 0x[] 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 freelist allocate 0x3e7000~33000 2019-12-07T19:34:04.768+0000 7fe2ca26b700 20 freelist _xor first_key 0x380000 last_key 0x400000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) get_onode oid 5#1:cd22b62d:::smithi13140796-27:head# key 0x858000000000000001cd22b6'-!smithi13140796-27!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 prepare 2019-12-07T19:34:04.768+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8417a600 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) r -2 v.len 0 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 5#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) get_onode oid 5#1:cd22b62d:::smithi13140796-27:head# key 0x858000000000000001cd22b6'-!smithi13140796-27!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) r -2 v.len 0 2019-12-07T19:34:04.768+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.3s5_head 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# -> 5#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.3s5_head 5#-3:cd22b62d:::temp_recovering_1.3s5_23'10_26_head:head# -> 5#1:cd22b62d:::smithi13140796-27:head# = 0 2019-12-07T19:34:04.768+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x0000000000000461'.missing/0000000000000001.3B44D64B.head.smithi13140796-27..' <- missing/0000000000000001.3B44D64B.head.smithi13140796-27.. 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:04.768+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1b00 cost 127707 (11 ios * 4000 + 83707 bytes) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1b00 onodes 0x558e84242f00 shared_blobs 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84243008) update 5#1:cd22b62d:::smithi13140796-27:head# 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84243008) update inline shard 801 bytes from 42 extents 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:cd22b62d:::smithi13140796-27:head# is 1292 (485 bytes onode + 2 bytes spanning blobs + 805 bytes inline extents) 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1b00 allocated 0x[41a000~14000] released 0x[] 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 freelist allocate 0x41a000~14000 2019-12-07T19:34:04.768+0000 7fe2c6263700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 prepare 2019-12-07T19:34:04.768+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e819a1b00 2019-12-07T19:34:04.769+0000 7fe2c6263700 10 osd.0 27 dequeue_op 0x558e81942f20 finish 2019-12-07T19:34:04.769+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.769+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 aio_wait 2019-12-07T19:34:04.769+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1b00 2019-12-07T19:34:04.769+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 io_done 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2687 < min_flush_size 524288 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84242f00 had 1 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402f900 had 1 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2710 < min_flush_size 524288 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2710 < min_flush_size 524288 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xcd5b7~a96 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xcd5b7 0xcd5b7~a96 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9d5b7 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x5b7 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x4d and padding block with 0xfb3 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xce04d 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000197002s (0.000036456s flush + 0.000160546s kv commit) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1b00> 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.769+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 kv_submitted 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1b00 2019-12-07T19:34:04.769+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 finishing 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1b00 onodes 0x558e84242f00 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7a8d8 in 0x558e802196c0) _finish_write discard buffer(0x558e843bb560 space 0x558e81a7a8d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7a9b8 in 0x558e802196c0) _finish_write discard buffer(0x558e843bb950 space 0x558e81a7a9b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7aa98 in 0x558e802196c0) _finish_write discard buffer(0x558e843d2750 space 0x558e81a7aa98 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7ada8 in 0x558e802196c0) _finish_write discard buffer(0x558e843d27e0 space 0x558e81a7ada8 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7ae88 in 0x558e802196c0) _finish_write discard buffer(0x558e843bb680 space 0x558e81a7ae88 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7af68 in 0x558e802196c0) _finish_write discard buffer(0x558e843bb710 space 0x558e81a7af68 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7b048 in 0x558e802196c0) _finish_write discard buffer(0x558e843bb7a0 space 0x558e81a7b048 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7b128 in 0x558e802196c0) _finish_write discard buffer(0x558e843bb830 space 0x558e81a7b128 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7b208 in 0x558e802196c0) _finish_write discard buffer(0x558e843bb8c0 space 0x558e81a7b208 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423c478 in 0x558e802196c0) _finish_write discard buffer(0x558e843bb5f0 space 0x558e8423c478 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1b00 done 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:04.769+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1b00 [] 2019-12-07T19:34:04.769+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.769+0000 7fe2ca26b700 10 osd.0 27 dequeue_op 0x558e81943b80 finish 2019-12-07T19:34:04.769+0000 7fe2ca26b700 20 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 23'9 active mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.769+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 lcod 23'9 active mbc={} ps=[2~1]] recovery_committed_to version 23'10 now ondisk 2019-12-07T19:34:04.769+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_trim(1.3s0 23'10 e27/27) v2 -- 0x558e84198960 con 0x558e818fe400 2019-12-07T19:34:04.769+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGPushReply(1.3s0 27/26 [PushReplyOp(1:cd22b62d:::smithi13140796-27:head)]) v3 -- 0x558e818a9400 con 0x558e818fe400 2019-12-07T19:34:04.769+0000 7fe2ca26b700 20 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'8 active m=2 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.769+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.769+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 aio_wait 2019-12-07T19:34:04.769+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417a600 2019-12-07T19:34:04.769+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 io_done 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2337 < min_flush_size 524288 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84243900 had 1 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2360 < min_flush_size 524288 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2360 < min_flush_size 524288 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xce04d~938 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xce04d 0xce04d~938 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9e04d 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x4d 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x985 and padding block with 0x67b 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xce985 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.769+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000111516s (0.000032978s flush + 0.000078538s kv commit) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.769+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417a600> 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.769+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 kv_submitted 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417a600 2019-12-07T19:34:04.769+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 finishing 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417a600 onodes 0x558e84243900 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edf2e8 in 0x558e80219c00) _finish_write discard buffer(0x558e843ba240 space 0x558e83edf2e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84014be8 in 0x558e80219c00) _finish_write discard buffer(0x558e843ba1b0 space 0x558e84014be8 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84462018 in 0x558e80219c00) _finish_write discard buffer(0x558e843ba2d0 space 0x558e84462018 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e844620f8 in 0x558e80219c00) _finish_write discard buffer(0x558e843ba360 space 0x558e844620f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e844621d8 in 0x558e80219c00) _finish_write discard buffer(0x558e843ba3f0 space 0x558e844621d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e844622b8 in 0x558e80219c00) _finish_write discard buffer(0x558e843ba480 space 0x558e844622b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84462398 in 0x558e80219c00) _finish_write discard buffer(0x558e843ba510 space 0x558e84462398 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84462478 in 0x558e80219c00) _finish_write discard buffer(0x558e843ba5a0 space 0x558e84462478 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84462558 in 0x558e80219c00) _finish_write discard buffer(0x558e843bb9e0 space 0x558e84462558 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84462638 in 0x558e80219c00) _finish_write discard buffer(0x558e843bba70 space 0x558e84462638 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84462718 in 0x558e80219c00) _finish_write discard buffer(0x558e843bbb00 space 0x558e84462718 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e844627f8 in 0x558e80219c00) _finish_write discard buffer(0x558e843bbb90 space 0x558e844627f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e844628d8 in 0x558e80219c00) _finish_write discard buffer(0x558e843bbc20 space 0x558e844628d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e844629b8 in 0x558e80219c00) _finish_write discard buffer(0x558e843bbcb0 space 0x558e844629b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'8 active m=2 mbc={} ps=[2~1]] recovery_committed_to version 21'9 now ondisk 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84462a98 in 0x558e80219c00) _finish_write discard buffer(0x558e843bbd40 space 0x558e84462a98 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84462b78 in 0x558e80219c00) _finish_write discard buffer(0x558e843bbdd0 space 0x558e84462b78 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84462c58 in 0x558e80219c00) _finish_write discard buffer(0x558e843bbe60 space 0x558e84462c58 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84462d38 in 0x558e80219c00) _finish_write discard buffer(0x558e843bbef0 space 0x558e84462d38 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84462e18 in 0x558e80219c00) _finish_write discard buffer(0x558e81170d80 space 0x558e84462e18 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84462ef8 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2000 space 0x558e84462ef8 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84462fd8 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2090 space 0x558e84462fd8 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGPushReply(1.bs0 27/26 [PushReplyOp(1:dce63cc3:::smithi13140796-50:head)]) v3 -- 0x558e818a8a00 con 0x558e818fe400 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e844630b8 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2120 space 0x558e844630b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84463198 in 0x558e80219c00) _finish_write discard buffer(0x558e843d21b0 space 0x558e84463198 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84463278 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2240 space 0x558e84463278 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84463358 in 0x558e80219c00) _finish_write discard buffer(0x558e843d22d0 space 0x558e84463358 0x0~2000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84463438 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2360 space 0x558e84463438 0x0~1000 writing nocache) 2019-12-07T19:34:04.769+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a600 done 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:04.769+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417a600 [] 2019-12-07T19:34:04.769+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.769+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.770+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 95 ==== MRecoveryReserve(1.3s5 RELEASE e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e84198960 con 0x558e818fe400 2019-12-07T19:34:04.770+0000 7fe2e6d3a700 15 osd.0 27 enqueue_peering_evt 1.3s5 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.770+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.770+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.770+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.770+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.770+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558e81231000 2019-12-07T19:34:04.770+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.770+0000 7fe2c6263700 5 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.770914+0000 1 0.000046 2019-12-07T19:34:04.770+0000 7fe2c6263700 5 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.770+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:04.770+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.770+0000 7fe2cea74700 15 osd.0 27 enqueue_peering_evt 1.5s4 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.770+0000 7fe2cea74700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.770+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.770+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.770+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.770+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e83d07000 2019-12-07T19:34:04.770+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active m=2 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.770+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MRecoveryReserve(1.5s0 GRANT e27/27) v3 -- 0x558e841992c0 con 0x558e818d6000 2019-12-07T19:34:04.770+0000 7fe2c9269700 5 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active m=2 mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.771006+0000 1 0.000035 2019-12-07T19:34:04.770+0000 7fe2c9269700 5 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active m=2 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.770+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:04.770+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.770+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 96 ==== MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=52, to_read={1:d313ddd1:::smithi13140796-49:head=0,262144,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e84243180 con 0x558e818fe400 2019-12-07T19:34:04.770+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e81943340 prio 3 cost 0 latency 0.000023 epoch 27 MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=52, to_read={1:d313ddd1:::smithi13140796-49:head=0,262144,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.770+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=52, to_read={1:d313ddd1:::smithi13140796-49:head=0,262144,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.770+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.770+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=52, to_read={1:d313ddd1:::smithi13140796-49:head=0,262144,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.770+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.770+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGOpItem(op=MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=52, to_read={1:d313ddd1:::smithi13140796-49:head=0,262144,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e81928000 2019-12-07T19:34:04.770+0000 7fe2ca26b700 10 osd.0 27 dequeue_op 0x558e81943340 prio 3 cost 0 latency 0.000071 MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=52, to_read={1:d313ddd1:::smithi13140796-49:head=0,262144,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.bs3 27/26 ECSubRead(tid=52, to_read={1:d313ddd1:::smithi13140796-49:head=0,262144,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.771+0000 7fe2ca26b700 25 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.771+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~40000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~40000 size 0x32000 (204800) 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e700 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9ce0 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194ecb0 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ee00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194ed20 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194eee0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194ef50 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64ee0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0fd50 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fab0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194fb20 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f9d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e7e0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d1880 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d09a0 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867490 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d02460 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d024d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd2000 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5c00 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac48c0 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5c70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9110 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64c40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64cb0 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c650a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014d90 blob([0x2be000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e84014e00 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840152d0 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015260 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84015340 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840153b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6c770 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c7e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6c850 blob([0x2e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c8c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6c930 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c9a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6ca10 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6ca80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6caf0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8af0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9d50 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840151f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84015180 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cb60 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6cbd0 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cc40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6ccb0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cd20 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6cd90 blob([!~1000,0x2f1000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558e83e6ce00 sbid 0x0)) need 0x1000~1000 cache has 0x[] 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e811d09a0 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867490 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e9110 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64c40 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e9d50 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840151f0 sbid 0x0)) need {<0x0, 0x2000> : [0x29000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x29000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194e700 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9ce0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194e7e0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d1880 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194ecb0 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ee00 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194ed20 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194eee0 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194ef50 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64ee0 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194fb20 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f9d0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac48c0 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5c70 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83c64cb0 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c650a0 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d02460 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d024d0 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0fd50 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fab0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83dd2000 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5c00 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6c770 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c7e0 sbid 0x0)) need {<0x0, 0x2000> : [0x1f000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1f000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6c850 blob([0x2e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c8c0 sbid 0x0)) need {<0x0, 0x2000> : [0x21000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x21000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6c930 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c9a0 sbid 0x0)) need {<0x0, 0x2000> : [0x23000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x23000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6ca10 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6ca80 sbid 0x0)) need {<0x0, 0x2000> : [0x25000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x25000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6caf0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8af0 sbid 0x0)) need {<0x0, 0x2000> : [0x27000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x27000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6cbd0 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cc40 sbid 0x0)) need {<0x0, 0x2000> : [0x2d000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2d000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6ccb0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cd20 sbid 0x0)) need {<0x0, 0x2000> : [0x2f000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2f000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6cd90 blob([!~1000,0x2f1000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558e83e6ce00 sbid 0x0)) need {<0x1000, 0x1000> : [0x31000:1000~1000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x31000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84014d90 blob([0x2be000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e84014e00 sbid 0x0)) need {<0x0, 0x1000> : [0x1a000:0~1000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84015180 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cb60 sbid 0x0)) need {<0x0, 0x2000> : [0x2b000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2b000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e840152d0 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015260 sbid 0x0)) need {<0x0, 0x2000> : [0x1b000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1b000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84015340 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840153b0 sbid 0x0)) need {<0x0, 0x2000> : [0x1d000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1d000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e811d09a0 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867490 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e818674a8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e84378900 space 0x558e818674a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e9110 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64c40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83c64c58 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d2510 space 0x558e83c64c58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e9d50 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840151f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x29000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015208 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d25a0 space 0x558e84015208 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194e700 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9ce0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e817e9cf8 in 0x558e80219c00) _discard 0x1000~1000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d2630 space 0x558e817e9cf8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194e7e0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d1880 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e811d1898 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d26c0 space 0x558e811d1898 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194ecb0 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ee00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194ee18 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d2240 space 0x558e8194ee18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194ed20 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194eee0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194eef8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d22d0 space 0x558e8194eef8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194ef50 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64ee0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83c64ef8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d3200 space 0x558e83c64ef8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194fb20 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f9d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194f9e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d3290 space 0x558e8194f9e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac48c0 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5c70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5c88 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d3320 space 0x558e83ac5c88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83c64cb0 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c650a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83c650b8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d33b0 space 0x558e83c650b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d02460 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d024d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83d024e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d3440 space 0x558e83d024e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0fd50 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fab0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194fac8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d34d0 space 0x558e8194fac8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83dd2000 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5c00 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5c18 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d3560 space 0x558e83ac5c18 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6c770 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c7e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1f000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c7f8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d35f0 space 0x558e83e6c7f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6c850 blob([0x2e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c8c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x21000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c8d8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d3680 space 0x558e83e6c8d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6c930 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c9a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x23000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c9b8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d3710 space 0x558e83e6c9b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6ca10 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6ca80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x25000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6ca98 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d37a0 space 0x558e83e6ca98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6caf0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8af0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x27000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e817e8b08 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d3830 space 0x558e817e8b08 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6cbd0 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cc40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2d000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6cc58 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d38c0 space 0x558e83e6cc58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6ccb0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cd20 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2f000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6cd38 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d3950 space 0x558e83e6cd38 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6cd90 blob([!~1000,0x2f1000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558e83e6ce00 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x31000:1000~1000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6ce18 in 0x558e80219c00) _discard 0x1000~1000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d39e0 space 0x558e83e6ce18 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84014d90 blob([0x2be000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e84014e00 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1a000:0~1000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84014e18 in 0x558e80219c00) _discard 0x0~1000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d3a70 space 0x558e84014e18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84015180 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cb60 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2b000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6cb78 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d3b00 space 0x558e83e6cb78 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e840152d0 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015260 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1b000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015278 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d3b90 space 0x558e84015278 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84015340 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840153b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1d000:0~2000]} 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e840153c8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 _add level 1 near 0 on buffer(0x558e843d3c20 space 0x558e840153c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.771+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0~40000 = 204800 2019-12-07T19:34:04.771+0000 7fe2ca26b700 20 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=204800 len=204800 2019-12-07T19:34:04.771+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:04.771+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:04.771+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:04.772+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 204800 2019-12-07T19:34:04.772+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key 2019-12-07T19:34:04.772+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# hinfo_key = 0 2019-12-07T19:34:04.772+0000 7fe2ca26b700 20 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:04.772+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 27/26 ECSubReadReply(tid=52, attrs_read=0)) v2 -- 0x558e84314000 con 0x558e818fe400 2019-12-07T19:34:04.772+0000 7fe2ca26b700 10 osd.0 27 dequeue_op 0x558e81943340 finish 2019-12-07T19:34:04.772+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.773+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 32 ==== MOSDPGPush(1.5s4 27/26 [PushOp(1:a9dea763:::smithi13140796-12:head, version: 21'2, data_included: [0~49152], data_size: 49152, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a9dea763:::smithi13140796-12:head@21'2, size: 196608, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:196608, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 50214+0+0 (unknown 0 0 0) 0x558e8118f800 con 0x558e818d6000 2019-12-07T19:34:04.773+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e84176580 prio 127 cost 50152 latency 0.000027 epoch 27 MOSDPGPush(1.5s4 27/26 [PushOp(1:a9dea763:::smithi13140796-12:head, version: 21'2, data_included: [0~49152], data_size: 49152, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a9dea763:::smithi13140796-12:head@21'2, size: 196608, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:196608, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.773+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 27/26 [PushOp(1:a9dea763:::smithi13140796-12:head, version: 21'2, data_included: [0~49152], data_size: 49152, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a9dea763:::smithi13140796-12:head@21'2, size: 196608, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:196608, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 50152 e27) 2019-12-07T19:34:04.773+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.773+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 27/26 [PushOp(1:a9dea763:::smithi13140796-12:head, version: 21'2, data_included: [0~49152], data_size: 49152, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a9dea763:::smithi13140796-12:head@21'2, size: 196608, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:196608, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 50152 e27) queued 2019-12-07T19:34:04.773+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.773+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 27/26 [PushOp(1:a9dea763:::smithi13140796-12:head, version: 21'2, data_included: [0~49152], data_size: 49152, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a9dea763:::smithi13140796-12:head@21'2, size: 196608, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:196608, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 50152 e27) pg 0x558e83d07000 2019-12-07T19:34:04.773+0000 7fe2c5261700 10 osd.0 27 dequeue_op 0x558e84176580 prio 127 cost 50152 latency 0.000077 MOSDPGPush(1.5s4 27/26 [PushOp(1:a9dea763:::smithi13140796-12:head, version: 21'2, data_included: [0~49152], data_size: 49152, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a9dea763:::smithi13140796-12:head@21'2, size: 196608, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:196608, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active m=2 mbc={} ps=[2~1]] 2019-12-07T19:34:04.773+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active m=2 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.5s4 27/26 [PushOp(1:a9dea763:::smithi13140796-12:head, version: 21'2, data_included: [0~49152], data_size: 49152, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a9dea763:::smithi13140796-12:head@21'2, size: 196608, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:196608, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.773+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active m=2 mbc={} ps=[2~1]] on_local_recover: 1:a9dea763:::smithi13140796-12:head 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'1 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active m=2 mbc={} ps=[2~1]] got missing 1:a9dea763:::smithi13140796-12:head v 21'2 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'3 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 lcod 21'1 active m=1 mbc={} ps=[2~1]] last_complete now 21'3 log.complete_to 21'4 2019-12-07T19:34:04.774+0000 7fe2c5261700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e8195f800 seq 8 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.5s4_head 0x558e8403fe00) get_onode oid 4#1:a9dea763:::smithi13140796-12:head# key 0x848000000000000001a9dea7'c!smithi13140796-12!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.5s4_head 0x558e8403fe00) r -2 v.len 0 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 4#1:a9dea763:::smithi13140796-12:head# 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.5s4_head 0x558e8403fe00) get_onode oid 4#1:a9dea763:::smithi13140796-12:head# key 0x848000000000000001a9dea7'c!smithi13140796-12!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.5s4_head 0x558e8403fe00) r -2 v.len 0 2019-12-07T19:34:04.774+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.5s4_head 4#1:a9dea763:::smithi13140796-12:head# 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1134 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.5s4_head 4#1:a9dea763:::smithi13140796-12:head# = 0 2019-12-07T19:34:04.774+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.5s4_head 4#1:a9dea763:::smithi13140796-12:head# 0x0~c000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 4#1:a9dea763:::smithi13140796-12:head# 0x0~c000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~c000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8195f800 6 blobs 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 fbmap_alloc 0x558e80233300 allocate 0xc000/1000,c000,0 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x42e000~c000/1000,c000,0 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x42e000~c000] 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0fea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e817e9e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0fea0 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9e30 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.blob(0x558e83d0fea0) get_ref 0x0~2000 Blob(0x558e83d0fea0 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9e30 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.blob(0x558e83d0fea0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83d0fea0 blob([0x42e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9e30 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e817e9e48 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d0fdc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc51f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d0fdc0 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc51f0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.blob(0x558e83d0fdc0) get_ref 0x0~2000 Blob(0x558e83d0fdc0 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc51f0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.blob(0x558e83d0fdc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83d0fdc0 blob([0x430000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc51f0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc5208 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc5110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc5110 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5260 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc5110) get_ref 0x0~2000 Blob(0x558e83cc5110 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5260 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc5110) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83cc5110 blob([0x432000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5260 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc5278 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83cc52d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83cc52d0 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5340 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc52d0) get_ref 0x0~2000 Blob(0x558e83cc52d0 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83cc5340 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.blob(0x558e83cc52d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83cc52d0 blob([0x434000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5340 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83cc5358 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84190af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84190a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84190af0 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84190a80 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.blob(0x558e84190af0) get_ref 0x0~2000 Blob(0x558e84190af0 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84190a80 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.blob(0x558e84190af0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e84190af0 blob([0x436000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84190a80 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e84190a98 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84190a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841909a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84190a10 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841909a0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.blob(0x558e84190a10) get_ref 0x0~2000 Blob(0x558e84190a10 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841909a0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.blob(0x558e84190a10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e84190a10 blob([0x438000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841909a0 sbid 0x0)) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e841909b8 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0xc000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.extentmap(0x558e83cb2888) dirty_range mark inline shard dirty 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.5s4_head 4#1:a9dea763:::smithi13140796-12:head# 0x0~c000 = 0 2019-12-07T19:34:04.774+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.5s4_head 4#1:a9dea763:::smithi13140796-12:head# 4 keys 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.5s4_head 4#1:a9dea763:::smithi13140796-12:head# 4 keys = 0 2019-12-07T19:34:04.774+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x0000000000000462'.missing/0000000000000001.59B75E6C.head.smithi13140796-12..' <- missing/0000000000000001.59B75E6C.head.smithi13140796-12.. 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.774+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195f800 cost 79559 (7 ios * 4000 + 51559 bytes) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195f800 onodes 0x558e83cb2780 shared_blobs 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.extentmap(0x558e83cb2888) update 4#1:a9dea763:::smithi13140796-12:head# 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore.extentmap(0x558e83cb2888) update inline shard 117 bytes from 6 extents 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 4#1:a9dea763:::smithi13140796-12:head# is 608 (485 bytes onode + 2 bytes spanning blobs + 121 bytes inline extents) 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195f800 allocated 0x[42e000~c000] released 0x[] 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 freelist allocate 0x42e000~c000 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 prepare 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8195f800 2019-12-07T19:34:04.774+0000 7fe2c5261700 10 osd.0 27 dequeue_op 0x558e84176580 finish 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.774+0000 7fe2c9269700 20 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'3 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 lcod 21'1 active m=1 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.774+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.774+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 aio_wait 2019-12-07T19:34:04.774+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195f800 2019-12-07T19:34:04.774+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 io_done 2019-12-07T19:34:04.774+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.774+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.774+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.774+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2101 < min_flush_size 524288 2019-12-07T19:34:04.774+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2780 had 1 2019-12-07T19:34:04.774+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:04.774+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2124 < min_flush_size 524288 2019-12-07T19:34:04.774+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2124 < min_flush_size 524288 2019-12-07T19:34:04.774+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.774+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xce985~84c to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.774+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xce985 0xce985~84c to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.774+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.774+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9e985 2019-12-07T19:34:04.774+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x985 2019-12-07T19:34:04.774+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.774+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x1d1 and padding block with 0xe2f 2019-12-07T19:34:04.774+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xcf1d1 2019-12-07T19:34:04.774+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.774+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.774+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.774+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.774+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000165094s (0.000039235s flush + 0.000125859s kv commit) 2019-12-07T19:34:04.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.774+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195f800> 2019-12-07T19:34:04.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.774+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 kv_submitted 2019-12-07T19:34:04.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195f800 2019-12-07T19:34:04.774+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.774+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 finishing 2019-12-07T19:34:04.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195f800 onodes 0x558e83cb2780 2019-12-07T19:34:04.774+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e9e48 in 0x558e80219180) _finish_write discard buffer(0x558e840322d0 space 0x558e817e9e48 0x0~2000 writing nocache) 2019-12-07T19:34:04.774+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc5208 in 0x558e80219180) _finish_write discard buffer(0x558e83c8fdd0 space 0x558e83cc5208 0x0~2000 writing nocache) 2019-12-07T19:34:04.774+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc5278 in 0x558e80219180) _finish_write discard buffer(0x558e83c8f7a0 space 0x558e83cc5278 0x0~2000 writing nocache) 2019-12-07T19:34:04.774+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83cc5358 in 0x558e80219180) _finish_write discard buffer(0x558e83c8e7e0 space 0x558e83cc5358 0x0~2000 writing nocache) 2019-12-07T19:34:04.774+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841909b8 in 0x558e80219180) _finish_write discard buffer(0x558e81a46000 space 0x558e841909b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.774+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84190a98 in 0x558e80219180) _finish_write discard buffer(0x558e83c8f0e0 space 0x558e84190a98 0x0~2000 writing nocache) 2019-12-07T19:34:04.774+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'3 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 lcod 21'1 active m=1 mbc={} ps=[2~1]] recovery_committed_to version 21'3 now ondisk 2019-12-07T19:34:04.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f800 done 2019-12-07T19:34:04.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:04.774+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195f800 [] 2019-12-07T19:34:04.774+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.774+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.774+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDPGPushReply(1.5s0 27/26 [PushReplyOp(1:a9dea763:::smithi13140796-12:head)]) v3 -- 0x558e84053600 con 0x558e818d6000 2019-12-07T19:34:04.774+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.774+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 97 ==== MOSDPGPush(1.bs1 27/26 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 205862+0+0 (unknown 0 0 0) 0x558e818a9400 con 0x558e818fe400 2019-12-07T19:34:04.774+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e841774a0 prio 127 cost 205800 latency 0.000024 epoch 27 MOSDPGPush(1.bs1 27/26 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.774+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 27/26 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 205800 e27) 2019-12-07T19:34:04.774+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.774+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 27/26 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 205800 e27) queued 2019-12-07T19:34:04.774+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.774+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGPush(1.bs1 27/26 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 205800 e27) pg 0x558e83c78000 2019-12-07T19:34:04.774+0000 7fe2c6263700 10 osd.0 27 dequeue_op 0x558e841774a0 prio 127 cost 205800 latency 0.000072 MOSDPGPush(1.bs1 27/26 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.bs1( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=2 mbc={} ps=[2~1]] 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=2 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.bs1 27/26 [PushOp(1:d313ddd1:::smithi13140796-49:head, version: 21'10, data_included: [0~204800], data_size: 204800, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:d313ddd1:::smithi13140796-49:head@21'10, size: 819200, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:819200, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=2 mbc={} ps=[2~1]] on_local_recover: 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'9 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=2 mbc={} ps=[2~1]] got missing 1:d313ddd1:::smithi13140796-49:head v 21'10 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'9 active m=1 mbc={} ps=[2~1]] last_complete now 21'11 log.complete_to 21'12 2019-12-07T19:34:04.775+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e83c56600 seq 6 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) get_onode oid 1#1:d313ddd1:::smithi13140796-49:head# key 0x818000000000000001d313ddd121736d'ithi13140796-49!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) r -2 v.len 0 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 1#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) get_onode oid 1#1:d313ddd1:::smithi13140796-49:head# key 0x818000000000000001d313ddd121736d'ithi13140796-49!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) r -2 v.len 0 2019-12-07T19:34:04.775+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1135 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# = 0 2019-12-07T19:34:04.775+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 1#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~32000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83c56600 25 blobs 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate 0x32000/1000,32000,0 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x43a000~32000/1000,32000,0 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x43a000~32000] 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84014af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84014af0 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b5e0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84014af0) get_ref 0x0~2000 Blob(0x558e84014af0 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b5e0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84014af0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e84014af0 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7b5e0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a7b5f8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7b490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84190f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7b490 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84190f50 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7b490) get_ref 0x0~2000 Blob(0x558e81a7b490 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84190f50 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7b490) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e81a7b490 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84190f50 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84190f68 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84190fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84191030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84190fc0 blob([0x43e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191030 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84190fc0) get_ref 0x0~2000 Blob(0x558e84190fc0 blob([0x43e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191030 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84190fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e84190fc0 blob([0x43e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191030 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191048 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841910a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84191110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841910a0 blob([0x440000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191110 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e841910a0) get_ref 0x0~2000 Blob(0x558e841910a0 blob([0x440000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191110 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e841910a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e841910a0 blob([0x440000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191110 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191128 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84191180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841911f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84191180 blob([0x442000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841911f0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191180) get_ref 0x0~2000 Blob(0x558e84191180 blob([0x442000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841911f0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191180) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e84191180 blob([0x442000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841911f0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191208 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84191260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841912d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84191260 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841912d0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191260) get_ref 0x0~2000 Blob(0x558e84191260 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841912d0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191260) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e84191260 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841912d0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841912e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84191340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841913b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84191340 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841913b0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191340) get_ref 0x0~2000 Blob(0x558e84191340 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841913b0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191340) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e84191340 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841913b0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841913c8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84191420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84191490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84191420 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191490 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191420) get_ref 0x0~2000 Blob(0x558e84191420 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191490 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191420) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e84191420 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191490 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841914a8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84191500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84191570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84191500 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191570 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191500) get_ref 0x0~2000 Blob(0x558e84191500 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191570 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191500) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e84191500 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191570 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191588 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841915e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84191650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841915e0 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191650 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e841915e0) get_ref 0x0~2000 Blob(0x558e841915e0 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191650 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e841915e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e841915e0 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191650 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191668 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841916c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84191730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841916c0 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191730 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e841916c0) get_ref 0x0~2000 Blob(0x558e841916c0 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191730 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e841916c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e841916c0 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191730 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191748 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841917a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84191810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841917a0 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191810 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e841917a0) get_ref 0x0~2000 Blob(0x558e841917a0 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191810 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e841917a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e841917a0 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191810 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191828 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84191880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841918f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84191880 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841918f0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191880) get_ref 0x0~2000 Blob(0x558e84191880 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841918f0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191880) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e84191880 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841918f0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191908 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84191960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841919d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84191960 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841919d0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191960) get_ref 0x0~2000 Blob(0x558e84191960 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841919d0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191960) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e84191960 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841919d0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841919e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84191a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84191ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84191a40 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191ab0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191a40) get_ref 0x0~2000 Blob(0x558e84191a40 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191ab0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191a40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e84191a40 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191ab0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191ac8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84191b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84191b90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84191b20 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191b90 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191b20) get_ref 0x0~2000 Blob(0x558e84191b20 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191b90 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191b20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e84191b20 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191b90 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191ba8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84191c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84191c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84191c00 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191c70 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191c00) get_ref 0x0~2000 Blob(0x558e84191c00 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191c70 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191c00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e84191c00 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191c70 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191c88 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84191ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84191d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84191ce0 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191d50 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191ce0) get_ref 0x0~2000 Blob(0x558e84191ce0 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191d50 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e84191ce0 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191d50 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191d68 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84191dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84191e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84191dc0 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191e30 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191dc0) get_ref 0x0~2000 Blob(0x558e84191dc0 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191e30 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e84191dc0 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191e30 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191e48 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84191ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84191f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84191ea0 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191f10 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191ea0) get_ref 0x0~2000 Blob(0x558e84191ea0 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84191f10 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e84191ea0 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191f10 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191f28 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84191f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a48000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84191f80 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a48000 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191f80) get_ref 0x0~2000 Blob(0x558e84191f80 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a48000 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e84191f80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e84191f80 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a48000 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a48018 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a48070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a480e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a48070 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a480e0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e81a48070) get_ref 0x0~2000 Blob(0x558e81a48070 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a480e0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e81a48070) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558e81a48070 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a480e0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a480f8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a48150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a481c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a48150 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a481c0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e81a48150) get_ref 0x0~2000 Blob(0x558e81a48150 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a481c0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e81a48150) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558e81a48150 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a481c0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a481d8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a48230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a482a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a48230 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a482a0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e81a48230) get_ref 0x0~2000 Blob(0x558e81a48230 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a482a0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e81a48230) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558e81a48230 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a482a0 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a482b8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a48310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a48380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a48310 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a48380 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e81a48310) get_ref 0x0~2000 Blob(0x558e81a48310 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a48380 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.blob(0x558e81a48310) get_ref init 0x2000, 1000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558e81a48310 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a48380 sbid 0x0)) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a48398 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x32000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84243c88) dirty_range mark inline shard dirty 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 = 0 2019-12-07T19:34:04.775+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 4 keys 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 4 keys = 0 2019-12-07T19:34:04.775+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x0000000000000460'.missing/0000000000000001.BC8CBBB8.head.smithi13140796-49..' <- missing/0000000000000001.BC8CBBB8.head.smithi13140796-49.. 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.775+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56600 cost 311037 (26 ios * 4000 + 207037 bytes) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56600 onodes 0x558e84243b80 shared_blobs 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84243c88) update 1#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84243c88) update inline shard 478 bytes from 25 extents 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:d313ddd1:::smithi13140796-49:head# is 969 (485 bytes onode + 2 bytes spanning blobs + 482 bytes inline extents) 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56600 allocated 0x[43a000~32000] released 0x[] 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 freelist allocate 0x43a000~32000 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 freelist _xor first_key 0x400000 last_key 0x400000 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 prepare 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83c56600 2019-12-07T19:34:04.775+0000 7fe2c6263700 10 osd.0 27 dequeue_op 0x558e841774a0 finish 2019-12-07T19:34:04.775+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.775+0000 7fe2ca26b700 20 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'9 active m=1 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.775+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.776+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 aio_wait 2019-12-07T19:34:04.776+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56600 2019-12-07T19:34:04.776+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 io_done 2019-12-07T19:34:04.776+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.776+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.776+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.776+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2292 < min_flush_size 524288 2019-12-07T19:34:04.776+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84243b80 had 1 2019-12-07T19:34:04.776+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:04.776+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2315 < min_flush_size 524288 2019-12-07T19:34:04.776+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2315 < min_flush_size 524288 2019-12-07T19:34:04.776+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.776+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xcf1d1~90b to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.776+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xcf1d1 0xcf1d1~90b to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.776+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.776+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9f1d1 2019-12-07T19:34:04.776+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x1d1 2019-12-07T19:34:04.776+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.776+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xadc and padding block with 0x524 2019-12-07T19:34:04.776+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xcfadc 2019-12-07T19:34:04.776+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.776+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.776+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.776+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.776+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000124164s (0.000040513s flush + 0.000083651s kv commit) 2019-12-07T19:34:04.776+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56600> 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.776+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 kv_submitted 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56600 2019-12-07T19:34:04.776+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 finishing 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56600 onodes 0x558e84243b80 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a48018 in 0x558e80219c00) _finish_write discard buffer(0x558e81a46630 space 0x558e81a48018 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a480f8 in 0x558e80219c00) _finish_write discard buffer(0x558e81a466c0 space 0x558e81a480f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a481d8 in 0x558e80219c00) _finish_write discard buffer(0x558e81a46750 space 0x558e81a481d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a482b8 in 0x558e80219c00) _finish_write discard buffer(0x558e81a467e0 space 0x558e81a482b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a48398 in 0x558e80219c00) _finish_write discard buffer(0x558e81a46870 space 0x558e81a48398 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7b5f8 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2900 space 0x558e81a7b5f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84190f68 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2990 space 0x558e84190f68 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84191048 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2a20 space 0x558e84191048 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84191128 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2ab0 space 0x558e84191128 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84191208 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2b40 space 0x558e84191208 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841912e8 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2bd0 space 0x558e841912e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841913c8 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2c60 space 0x558e841913c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'9 active m=1 mbc={} ps=[2~1]] recovery_committed_to version 21'11 now ondisk 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841914a8 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2cf0 space 0x558e841914a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84191588 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2d80 space 0x558e84191588 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84191668 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2e10 space 0x558e84191668 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84191748 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2ea0 space 0x558e84191748 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84191828 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2f30 space 0x558e84191828 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84191908 in 0x558e80219c00) _finish_write discard buffer(0x558e843d2fc0 space 0x558e84191908 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841919e8 in 0x558e80219c00) _finish_write discard buffer(0x558e843d3050 space 0x558e841919e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGPushReply(1.bs0 27/26 [PushReplyOp(1:d313ddd1:::smithi13140796-49:head)]) v3 -- 0x558e84065600 con 0x558e818fe400 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84191ac8 in 0x558e80219c00) _finish_write discard buffer(0x558e843d30e0 space 0x558e84191ac8 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84191ba8 in 0x558e80219c00) _finish_write discard buffer(0x558e843d3170 space 0x558e84191ba8 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84191c88 in 0x558e80219c00) _finish_write discard buffer(0x558e81a463f0 space 0x558e84191c88 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84191d68 in 0x558e80219c00) _finish_write discard buffer(0x558e81a46480 space 0x558e84191d68 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84191e48 in 0x558e80219c00) _finish_write discard buffer(0x558e81a46510 space 0x558e84191e48 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84191f28 in 0x558e80219c00) _finish_write discard buffer(0x558e81a465a0 space 0x558e84191f28 0x0~2000 writing nocache) 2019-12-07T19:34:04.776+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56600 done 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:04.776+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56600 [] 2019-12-07T19:34:04.776+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.776+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.776+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 98 ==== MOSDPGRecoveryDelete(1.bs1 e27,26 1:d07269f7:::smithi13140796-28:head,21'12) v2 ==== 123+0+0 (unknown 0 0 0) 0x558e84065600 con 0x558e818fe400 2019-12-07T19:34:04.776+0000 7fe2e6d3a700 15 osd.0 27 enqueue_op 0x558e84177b80 prio 3 cost 1000 latency 0.000014 epoch 27 MOSDPGRecoveryDelete(1.bs1 e27,26 1:d07269f7:::smithi13140796-28:head,21'12) v2 2019-12-07T19:34:04.776+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGRecoveryDelete(1.bs1 e27,26 1:d07269f7:::smithi13140796-28:head,21'12) v2) prio 3 cost 1000 e27) 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGRecoveryDelete(1.bs1 e27,26 1:d07269f7:::smithi13140796-28:head,21'12) v2) prio 3 cost 1000 e27) queued 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDPGRecoveryDelete(1.bs1 e27,26 1:d07269f7:::smithi13140796-28:head,21'12) v2) prio 3 cost 1000 e27) pg 0x558e83c78000 2019-12-07T19:34:04.777+0000 7fe2c6263700 10 osd.0 27 dequeue_op 0x558e84177b80 prio 3 cost 1000 latency 0.000119 MOSDPGRecoveryDelete(1.bs1 e27,26 1:d07269f7:::smithi13140796-28:head,21'12) v2 pg pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[2~1]] 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[2~1]] handle_recovery_delete 0x558e84177b80 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[2~1]] remove_missing_object 1:d07269f7:::smithi13140796-28:head 21'12 2019-12-07T19:34:04.777+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e83c56c00 seq 7 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) get_onode oid 1#1:d07269f7:::smithi13140796-28:head# key 0x818000000000000001d07269f721736d'ithi13140796-28!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.bs1_head 0x558e841981e0) r -2 v.len 0 2019-12-07T19:34:04.777+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 1#1:d07269f7:::smithi13140796-28:head# 2019-12-07T19:34:04.777+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56c00 cost 4212 (1 ios * 4000 + 212 bytes) 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56c00 onodes shared_blobs 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56c00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.777+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 prepare 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56c00 2019-12-07T19:34:04.777+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 io_done 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.777+0000 7fe2c6263700 10 osd.0 27 dequeue_op 0x558e84177b80 finish 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xcfadc~2e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xcfadc 0xcfadc~2e to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9fadc 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xadc 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb0a and padding block with 0x4f6 2019-12-07T19:34:04.777+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xcfb0a 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000092756s (0.000000087s flush + 0.000092669s kv commit) 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56c00> 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.777+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 kv_submitted 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56c00 2019-12-07T19:34:04.777+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 finishing 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56c00 onodes 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56c00 done 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:04.777+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56c00 [] 2019-12-07T19:34:04.777+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.777+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[2~1]] on_local_recover: 1:d07269f7:::smithi13140796-28:head 2019-12-07T19:34:04.777+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 lc 21'11 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active m=1 mbc={} ps=[2~1]] got missing 1:d07269f7:::smithi13140796-28:head v 21'12 2019-12-07T19:34:04.777+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'11 active mbc={} ps=[2~1]] last_complete now 21'12 log.complete_to at end 2019-12-07T19:34:04.777+0000 7fe2ca26b700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.777+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:04.777+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e8417ac00 seq 8 2019-12-07T19:34:04.777+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.777+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x0000000000000460'.missing/0000000000000001.B0E469FE.head.smithi13140796-28..' <- missing/0000000000000001.B0E469FE.head.smithi13140796-28.. 2019-12-07T19:34:04.777+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.777+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:04.777+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:04.777+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.777+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.777+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:04.777+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417ac00 cost 5409 (1 ios * 4000 + 1409 bytes) 2019-12-07T19:34:04.777+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417ac00 onodes shared_blobs 2019-12-07T19:34:04.777+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:04.777+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 prepare 2019-12-07T19:34:04.777+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417ac00 2019-12-07T19:34:04.777+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 io_done 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1185 < min_flush_size 524288 2019-12-07T19:34:04.777+0000 7fe2ca26b700 20 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'11 active mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.777+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1208 < min_flush_size 524288 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xcfb0a~4b8 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xcfb0a 0xcfb0a~4b8 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~a0000 x_off 0x9fb0a 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb0a 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xfc2 and padding block with 0x3e 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xcffc2 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.777+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000142438s (0.000000064s flush + 0.000142374s kv commit) 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.777+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417ac00> 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.777+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 kv_submitted 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417ac00 2019-12-07T19:34:04.777+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 finishing 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417ac00 onodes 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417ac00 done 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:04.777+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417ac00 [] 2019-12-07T19:34:04.777+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.777+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.777+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.777+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 lcod 21'11 active mbc={} ps=[2~1]] recovery_committed_to version 21'12 now ondisk 2019-12-07T19:34:04.777+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_trim(1.bs0 21'12 e27/27) v2 -- 0x558e819b1e00 con 0x558e818fe400 2019-12-07T19:34:04.777+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDPGRecoveryDeleteReply(1.bs0 e27,26 1:d07269f7:::smithi13140796-28:head,21'12) v2 -- 0x558e818a9400 con 0x558e818fe400 2019-12-07T19:34:04.777+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.778+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 99 ==== MRecoveryReserve(1.bs1 RELEASE e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e819b1e00 con 0x558e818fe400 2019-12-07T19:34:04.778+0000 7fe2e6d3a700 15 osd.0 27 enqueue_peering_evt 1.bs1 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.778+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.778+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.778+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.778+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.778+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558e83c78000 2019-12-07T19:34:04.778+0000 7fe2c6263700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.778+0000 7fe2c6263700 5 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.778876+0000 1 0.000036 2019-12-07T19:34:04.778+0000 7fe2cea74700 15 osd.0 27 enqueue_peering_evt 1.1s1 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.778+0000 7fe2cea74700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.778+0000 7fe2c6263700 5 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.778+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:04.778+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:04.778+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 33 ==== MOSDPGPush(1.5s4 27/26 [PushOp(1:a8debf31:::smithi13140796-24:head, version: 21'4, data_included: [0~98304], data_size: 98304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a8debf31:::smithi13140796-24:head@21'4, size: 393216, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:393216, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 99366+0+0 (unknown 0 0 0) 0x558e84053600 con 0x558e818d6000 2019-12-07T19:34:04.778+0000 7fe2e753b700 15 osd.0 27 enqueue_op 0x558e84176840 prio 127 cost 99304 latency 0.000033 epoch 27 MOSDPGPush(1.5s4 27/26 [PushOp(1:a8debf31:::smithi13140796-24:head, version: 21'4, data_included: [0~98304], data_size: 98304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a8debf31:::smithi13140796-24:head@21'4, size: 393216, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:393216, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.778+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 27/26 [PushOp(1:a8debf31:::smithi13140796-24:head, version: 21'4, data_included: [0~98304], data_size: 98304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a8debf31:::smithi13140796-24:head@21'4, size: 393216, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:393216, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 99304 e27) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 27/26 [PushOp(1:a8debf31:::smithi13140796-24:head, version: 21'4, data_included: [0~98304], data_size: 98304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a8debf31:::smithi13140796-24:head@21'4, size: 393216, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:393216, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 99304 e27) queued 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGOpItem(op=MOSDPGPush(1.5s4 27/26 [PushOp(1:a8debf31:::smithi13140796-24:head, version: 21'4, data_included: [0~98304], data_size: 98304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a8debf31:::smithi13140796-24:head@21'4, size: 393216, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:393216, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 99304 e27) pg 0x558e83d07000 2019-12-07T19:34:04.778+0000 7fe2c5261700 10 osd.0 27 dequeue_op 0x558e84176840 prio 127 cost 99304 latency 0.000081 MOSDPGPush(1.5s4 27/26 [PushOp(1:a8debf31:::smithi13140796-24:head, version: 21'4, data_included: [0~98304], data_size: 98304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a8debf31:::smithi13140796-24:head@21'4, size: 393216, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:393216, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.5s4( v 27'8 lc 21'3 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active m=1 mbc={} ps=[2~1]] 2019-12-07T19:34:04.778+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'3 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active m=1 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.5s4 27/26 [PushOp(1:a8debf31:::smithi13140796-24:head, version: 21'4, data_included: [0~98304], data_size: 98304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:a8debf31:::smithi13140796-24:head@21'4, size: 393216, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:393216, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.778+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'3 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active m=1 mbc={} ps=[2~1]] on_local_recover: 1:a8debf31:::smithi13140796-24:head 2019-12-07T19:34:04.778+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 lc 21'3 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active m=1 mbc={} ps=[2~1]] got missing 1:a8debf31:::smithi13140796-24:head v 21'4 2019-12-07T19:34:04.778+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 lcod 21'3 active mbc={} ps=[2~1]] last_complete now 27'8 log.complete_to at end 2019-12-07T19:34:04.778+0000 7fe2c5261700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.778+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e8195e300 seq 9 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.5s4_head 0x558e8403fe00) get_onode oid 4#1:a8debf31:::smithi13140796-24:head# key 0x848000000000000001a8debf'1!smithi13140796-24!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.5s4_head 0x558e8403fe00) r -2 v.len 0 2019-12-07T19:34:04.778+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 4#1:a8debf31:::smithi13140796-24:head# 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.5s4_head 0x558e8403fe00) get_onode oid 4#1:a8debf31:::smithi13140796-24:head# key 0x848000000000000001a8debf'1!smithi13140796-24!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.5s4_head 0x558e8403fe00) r -2 v.len 0 2019-12-07T19:34:04.778+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.5s4_head 4#1:a8debf31:::smithi13140796-24:head# 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1136 2019-12-07T19:34:04.778+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.5s4_head 4#1:a8debf31:::smithi13140796-24:head# = 0 2019-12-07T19:34:04.778+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.5s4_head 4#1:a8debf31:::smithi13140796-24:head# 0x0~18000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 4#1:a8debf31:::smithi13140796-24:head# 0x0~18000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.778+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~18000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8195e300 12 blobs 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.778+0000 7fe2c5261700 10 fbmap_alloc 0x558e80233300 allocate 0x18000/1000,18000,0 2019-12-07T19:34:04.778+0000 7fe2c5261700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x46c000~18000/1000,18000,0 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x46c000~18000] 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d023f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81079880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d023f0 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81079880 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e83d023f0) get_ref 0x0~2000 Blob(0x558e83d023f0 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81079880 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e83d023f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83d023f0 blob([0x46c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81079880 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e81079898 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83edf6c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d02e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83edf6c0 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02e00 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e83edf6c0) get_ref 0x0~2000 Blob(0x558e83edf6c0 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02e00 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e83edf6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83edf6c0 blob([0x46e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02e00 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83d02e18 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7b6c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7b6c0 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b650 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e81a7b6c0) get_ref 0x0~2000 Blob(0x558e81a7b6c0 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b650 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e81a7b6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e81a7b6c0 blob([0x470000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7b650 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e81a7b668 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a49c00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a49c70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a49c00 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49c70 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e81a49c00) get_ref 0x0~2000 Blob(0x558e81a49c00 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49c70 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e81a49c00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e81a49c00 blob([0x472000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a49c70 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e81a49c88 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a49ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a49d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a49ce0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49d50 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e81a49ce0) get_ref 0x0~2000 Blob(0x558e81a49ce0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49d50 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e81a49ce0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e81a49ce0 blob([0x474000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a49d50 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e81a49d68 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a49dc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a49e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a49dc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49e30 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e81a49dc0) get_ref 0x0~2000 Blob(0x558e81a49dc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49e30 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e81a49dc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e81a49dc0 blob([0x476000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a49e30 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e81a49e48 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a49ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a49f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a49ea0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49f10 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e81a49ea0) get_ref 0x0~2000 Blob(0x558e81a49ea0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49f10 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e81a49ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e81a49ea0 blob([0x478000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a49f10 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e81a49f28 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a49f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f60000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a49f80 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f60000 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e81a49f80) get_ref 0x0~2000 Blob(0x558e81a49f80 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f60000 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e81a49f80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e81a49f80 blob([0x47a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f60000 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83f60018 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f60070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f600e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f60070 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f600e0 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e83f60070) get_ref 0x0~2000 Blob(0x558e83f60070 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f600e0 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e83f60070) get_ref init 0x2000, 1000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83f60070 blob([0x47c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f600e0 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83f600f8 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f60150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f601c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f60150 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f601c0 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e83f60150) get_ref 0x0~2000 Blob(0x558e83f60150 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f601c0 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e83f60150) get_ref init 0x2000, 1000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83f60150 blob([0x47e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f601c0 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83f601d8 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f60230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f602a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f60230 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f602a0 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e83f60230) get_ref 0x0~2000 Blob(0x558e83f60230 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f602a0 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e83f60230) get_ref init 0x2000, 1000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83f60230 blob([0x480000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f602a0 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83f602b8 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f60310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f60380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f60310 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f60380 sbid 0x0)) 2019-12-07T19:34:04.778+0000 7fe2c5261700 20 bluestore.blob(0x558e83f60310) get_ref 0x0~2000 Blob(0x558e83f60310 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f60380 sbid 0x0)) 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore.blob(0x558e83f60310) get_ref init 0x2000, 1000 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83f60310 blob([0x482000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f60380 sbid 0x0)) 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore.BufferSpace(0x558e83f60398 in 0x558e80219180) _discard 0x0~2000 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x18000 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore.extentmap(0x558e84315008) dirty_range mark inline shard dirty 2019-12-07T19:34:04.779+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.5s4_head 4#1:a8debf31:::smithi13140796-24:head# 0x0~18000 = 0 2019-12-07T19:34:04.779+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.779+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.5s4_head 4#1:a8debf31:::smithi13140796-24:head# 4 keys 2019-12-07T19:34:04.779+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.779+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.779+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.5s4_head 4#1:a8debf31:::smithi13140796-24:head# 4 keys = 0 2019-12-07T19:34:04.779+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e819e4000 2019-12-07T19:34:04.779+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x0000000000000462'.missing/0000000000000001.51B7DFC8.head.smithi13140796-24..' <- missing/0000000000000001.51B7DFC8.head.smithi13140796-24.. 2019-12-07T19:34:04.779+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.779+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.779+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:04.779+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.779+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e300 cost 152711 (13 ios * 4000 + 100711 bytes) 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e300 onodes 0x558e84314f00 shared_blobs 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore.extentmap(0x558e84315008) update 4#1:a8debf31:::smithi13140796-24:head# 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore.extentmap(0x558e84315008) update inline shard 231 bytes from 12 extents 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 4#1:a8debf31:::smithi13140796-24:head# is 722 (485 bytes onode + 2 bytes spanning blobs + 235 bytes inline extents) 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e300 allocated 0x[46c000~18000] released 0x[] 2019-12-07T19:34:04.779+0000 7fe2c5261700 10 freelist allocate 0x46c000~18000 2019-12-07T19:34:04.779+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MRecoveryReserve(1.1s0 GRANT e27/27) v3 -- 0x558e84198000 con 0x558e818d7000 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 freelist _xor first_key 0x400000 last_key 0x480000 2019-12-07T19:34:04.779+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 prepare 2019-12-07T19:34:04.779+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8195e300 2019-12-07T19:34:04.779+0000 7fe2cb26d700 5 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.779751+0000 1 0.000019 2019-12-07T19:34:04.779+0000 7fe2cb26d700 5 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.779+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:04.779+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.779+0000 7fe2c5261700 10 osd.0 27 dequeue_op 0x558e84176840 finish 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.779+0000 7fe2c9269700 20 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 lcod 21'3 active mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.779+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.779+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 aio_wait 2019-12-07T19:34:04.779+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e300 2019-12-07T19:34:04.779+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 io_done 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 62 < min_flush_size 524288 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2255 < min_flush_size 524288 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84314f00 had 1 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2278 < min_flush_size 524288 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2278 < min_flush_size 524288 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xcffc2~8e6 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xcffc2 0xcffc2~8e6 to file(ino 16 size 0xd0000 mtime 2019-12-07T19:34:04.645660+0000 allocated d0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~a0000]) 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _allocate len 0x8a8 from 0 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _allocate len 0x8a8 from 1 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd200000 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd200000~10000/10000,10000,acd200000 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0xe0000 to include allocated 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 33 (was clean) 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0x9ffc2 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xfc2 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x8a8 and padding block with 0x758 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0xe0000 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xd08a8 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (33) on file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]), flushing log 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 33 len 0x49 crc 0xdac13ca) 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x20000~1000 to file(ino 1 size 0x20000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x20000 0x20000~1000 to file(ino 1 size 0x20000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x21000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x20000 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x21000 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:34:04.779+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000024 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 33 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000437725s (0.000045813s flush + 0.000391912s kv commit) 2019-12-07T19:34:04.779+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e300> 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.779+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 kv_submitted 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e300 2019-12-07T19:34:04.779+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 finishing 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e300 onodes 0x558e84314f00 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81079898 in 0x558e80219180) _finish_write discard buffer(0x558e81a461b0 space 0x558e81079898 0x0~2000 writing nocache) 2019-12-07T19:34:04.779+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a49c88 in 0x558e80219180) _finish_write discard buffer(0x558e81a46360 space 0x558e81a49c88 0x0~2000 writing nocache) 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a49d68 in 0x558e80219180) _finish_write discard buffer(0x558e81a46f30 space 0x558e81a49d68 0x0~2000 writing nocache) 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a49e48 in 0x558e80219180) _finish_write discard buffer(0x558e81a46fc0 space 0x558e81a49e48 0x0~2000 writing nocache) 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a49f28 in 0x558e80219180) _finish_write discard buffer(0x558e81a47050 space 0x558e81a49f28 0x0~2000 writing nocache) 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7b668 in 0x558e80219180) _finish_write discard buffer(0x558e81a462d0 space 0x558e81a7b668 0x0~2000 writing nocache) 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d02e18 in 0x558e80219180) _finish_write discard buffer(0x558e81a46240 space 0x558e83d02e18 0x0~2000 writing nocache) 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f60018 in 0x558e80219180) _finish_write discard buffer(0x558e81a470e0 space 0x558e83f60018 0x0~2000 writing nocache) 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f600f8 in 0x558e80219180) _finish_write discard buffer(0x558e81a47170 space 0x558e83f600f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f601d8 in 0x558e80219180) _finish_write discard buffer(0x558e81a47200 space 0x558e83f601d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.779+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 lcod 21'3 active mbc={} ps=[2~1]] recovery_committed_to version 27'8 now ondisk 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f602b8 in 0x558e80219180) _finish_write discard buffer(0x558e81a47290 space 0x558e83f602b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f60398 in 0x558e80219180) _finish_write discard buffer(0x558e81a47320 space 0x558e83f60398 0x0~2000 writing nocache) 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 done 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:04.779+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_trim(1.5s0 27'8 e27/27) v2 -- 0x558e841994a0 con 0x558e818d6000 2019-12-07T19:34:04.779+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e300 [] 2019-12-07T19:34:04.779+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.779+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.779+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- MOSDPGPushReply(1.5s0 27/26 [PushReplyOp(1:a8debf31:::smithi13140796-24:head)]) v3 -- 0x558e8118f800 con 0x558e818d6000 2019-12-07T19:34:04.779+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.780+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 34 ==== MRecoveryReserve(1.5s4 RELEASE e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e841994a0 con 0x558e818d6000 2019-12-07T19:34:04.780+0000 7fe2e753b700 15 osd.0 27 enqueue_peering_evt 1.5s4 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.780+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.780+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.780+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.780+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.780+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558e83d07000 2019-12-07T19:34:04.780+0000 7fe2c5261700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.780+0000 7fe2c5261700 5 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.781519+0000 1 0.000029 2019-12-07T19:34:04.780+0000 7fe2c5261700 5 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.780+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:04.780+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:04.780+0000 7fe2cea74700 15 osd.0 27 enqueue_peering_evt 1.9s3 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.780+0000 7fe2cea74700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.780+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.780+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.780+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.780+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e8192b000 2019-12-07T19:34:04.780+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.780+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.9s0 GRANT e27/27) v3 -- 0x558e8403e1e0 con 0x558e818d7c00 2019-12-07T19:34:04.780+0000 7fe2c7265700 5 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.781669+0000 1 0.000028 2019-12-07T19:34:04.780+0000 7fe2c7265700 5 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.781+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:04.781+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.781+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 91 ==== MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=76, to_read={1:891f0d13:::smithi13140796-14:4=0,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:4=[0,2,4,2]}, attrs_to_read=)) v3 ==== 249+0+0 (unknown 0 0 0) 0x558e811dc780 con 0x558e818d7000 2019-12-07T19:34:04.781+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84177760 prio 3 cost 0 latency 0.000022 epoch 27 MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=76, to_read={1:891f0d13:::smithi13140796-14:4=0,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:4=[0,2,4,2]}, attrs_to_read=)) v3 2019-12-07T19:34:04.781+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=76, to_read={1:891f0d13:::smithi13140796-14:4=0,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:4=[0,2,4,2]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=76, to_read={1:891f0d13:::smithi13140796-14:4=0,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:4=[0,2,4,2]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=76, to_read={1:891f0d13:::smithi13140796-14:4=0,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:4=[0,2,4,2]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e819e4000 2019-12-07T19:34:04.781+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e84177760 prio 3 cost 0 latency 0.000061 MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=76, to_read={1:891f0d13:::smithi13140796-14:4=0,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:4=[0,2,4,2]}, attrs_to_read=)) v3 pg pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.781+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=76, to_read={1:891f0d13:::smithi13140796-14:4=0,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:4=[0,2,4,2]}, attrs_to_read=)) v3 2019-12-07T19:34:04.781+0000 7fe2cb26d700 25 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.781+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x0~400 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~400 size 0xe000 (57344) 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6d5e0 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) need 0x0~400 cache has 0x[0~400] 2019-12-07T19:34:04.781+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x0~400 = 1024 2019-12-07T19:34:04.781+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x800~400 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x800~400 size 0xe000 (57344) 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6d5e0 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.781+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x800~400 = 1024 2019-12-07T19:34:04.781+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1000~400 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1000~400 size 0xe000 (57344) 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6d5e0 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6d5e0 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) need {<0x1000, 0x1000> : [0x1000:1000~400]} 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6d5e0 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) need 0x{<0x1000, 0x1000> : [0x1000:1000~400]} 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e83c64088 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e841ecea0 space 0x558e83c64088 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.781+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1000~400 = 1024 2019-12-07T19:34:04.781+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1800~400 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1800~400 size 0xe000 (57344) 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6d5e0 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.781+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1800~400 = 1024 2019-12-07T19:34:04.781+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2000~400 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2000~400 size 0xe000 (57344) 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6d570 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6d570 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) need {<0x0, 0x1000> : [0x2000:0~400]} 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6d570 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) need 0x{<0x0, 0x1000> : [0x2000:0~400]} 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e83c657b8 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81171680 space 0x558e83c657b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.781+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2000~400 = 1024 2019-12-07T19:34:04.781+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2800~400 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2800~400 size 0xe000 (57344) 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6d570 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.781+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2800~400 = 1024 2019-12-07T19:34:04.781+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3000~400 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3000~400 size 0xe000 (57344) 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6d570 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83e6d570 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) need {<0x1000, 0x1000> : [0x3000:1000~400]} 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83e6d570 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) need 0x{<0x1000, 0x1000> : [0x3000:1000~400]} 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e83c657b8 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81170ea0 space 0x558e83c657b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.781+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3000~400 = 1024 2019-12-07T19:34:04.781+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3800~400 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3800~400 size 0xe000 (57344) 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6d570 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.781+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3800~400 = 1024 2019-12-07T19:34:04.781+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x4000~400 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4000~400 size 0xe000 (57344) 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95d50 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95d50 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) need {<0x0, 0x1000> : [0x4000:0~400]} 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95d50 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) need 0x{<0x0, 0x1000> : [0x4000:0~400]} 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e83c65908 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e80229e60 space 0x558e83c65908 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.781+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x4000~400 = 1024 2019-12-07T19:34:04.781+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x4800~400 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4800~400 size 0xe000 (57344) 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95d50 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.781+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x4800~400 = 1024 2019-12-07T19:34:04.781+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x5000~400 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5000~400 size 0xe000 (57344) 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95d50 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95d50 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) need {<0x1000, 0x1000> : [0x5000:1000~400]} 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x5000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95d50 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) need 0x{<0x1000, 0x1000> : [0x5000:1000~400]} 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e83c65908 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e840330e0 space 0x558e83c65908 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.781+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x5000~400 = 1024 2019-12-07T19:34:04.781+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x5800~400 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5800~400 size 0xe000 (57344) 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95d50 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.781+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x5800~400 = 1024 2019-12-07T19:34:04.781+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x6000~400 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6000~400 size 0xe000 (57344) 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014070 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84014070 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) need {<0x0, 0x1000> : [0x6000:0~400]} 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.781+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84014070 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) need 0x{<0x0, 0x1000> : [0x6000:0~400]} 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e817e9ba8 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e84033170 space 0x558e817e9ba8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.782+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x6000~400 = 1024 2019-12-07T19:34:04.782+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x6800~400 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6800~400 size 0xe000 (57344) 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014070 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.782+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x6800~400 = 1024 2019-12-07T19:34:04.782+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x7000~400 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7000~400 size 0xe000 (57344) 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014070 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84014070 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) need {<0x1000, 0x1000> : [0x7000:1000~400]} 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x7000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84014070 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) need 0x{<0x1000, 0x1000> : [0x7000:1000~400]} 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e817e9ba8 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e84033320 space 0x558e817e9ba8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.782+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x7000~400 = 1024 2019-12-07T19:34:04.782+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x7800~400 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7800~400 size 0xe000 (57344) 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014070 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.782+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x7800~400 = 1024 2019-12-07T19:34:04.782+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x8000~400 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8000~400 size 0xe000 (57344) 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014380 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84014380 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) need {<0x0, 0x1000> : [0x8000:0~400]} 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84014380 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) need 0x{<0x0, 0x1000> : [0x8000:0~400]} 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e817e9668 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81a478c0 space 0x558e817e9668 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.782+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x8000~400 = 1024 2019-12-07T19:34:04.782+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x8800~400 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8800~400 size 0xe000 (57344) 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014380 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.782+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x8800~400 = 1024 2019-12-07T19:34:04.782+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x9000~400 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9000~400 size 0xe000 (57344) 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014380 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84014380 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) need {<0x1000, 0x1000> : [0x9000:1000~400]} 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x9000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84014380 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) need 0x{<0x1000, 0x1000> : [0x9000:1000~400]} 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e817e9668 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81a47950 space 0x558e817e9668 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.782+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x9000~400 = 1024 2019-12-07T19:34:04.782+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x9800~400 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9800~400 size 0xe000 (57344) 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014380 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.782+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x9800~400 = 1024 2019-12-07T19:34:04.782+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xa000~400 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa000~400 size 0xe000 (57344) 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840140e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e840140e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) need {<0x0, 0x1000> : [0xa000:0~400]} 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e840140e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) need 0x{<0x0, 0x1000> : [0xa000:0~400]} 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e817e96d8 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81a479e0 space 0x558e817e96d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.782+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xa000~400 = 1024 2019-12-07T19:34:04.782+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xa800~400 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa800~400 size 0xe000 (57344) 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840140e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.782+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xa800~400 = 1024 2019-12-07T19:34:04.782+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xb000~400 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb000~400 size 0xe000 (57344) 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840140e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e840140e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) need {<0x1000, 0x1000> : [0xb000:1000~400]} 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xb000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.782+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.783+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 163 ==== MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=119, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 263+0+0 (unknown 0 0 0) 0x558e83b09680 con 0x558e818d7c00 2019-12-07T19:34:04.783+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e84177600 prio 3 cost 0 latency 0.000024 epoch 27 MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=119, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.783+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=119, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=119, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=119, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e8192b000 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 osd.0 27 dequeue_op 0x558e84177600 prio 3 cost 0 latency 0.000066 MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=119, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=119, to_read={1:9b8adabe:::smithi13140796-6:head=0,262144,0}, subchunks={1:9b8adabe:::smithi13140796-6:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.783+0000 7fe2c7265700 25 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x200~200 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x200~200 = 512 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x600~200 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x600~200 = 512 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xa00~200 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xa00~200 = 512 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xe00~200 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xe00~200 = 512 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e840140e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) need 0x{<0x1000, 0x1000> : [0xb000:1000~400]} 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1200~200 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e817e96d8 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81a47a70 space 0x558e817e96d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xb000~400 = 1024 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xb800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e840140e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xb800~400 = 1024 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xc000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014230 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84014230 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) need {<0x0, 0x1000> : [0xc000:0~400]} 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84014230 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) need 0x{<0x0, 0x1000> : [0xc000:0~400]} 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e817e9c88 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81a47b00 space 0x558e817e9c88 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xc000~400 = 1024 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xc800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014230 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e818667f8 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xc800~400 = 1024 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e841ed830 space 0x558e818667f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xd000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1200~200 = 512 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014230 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1600~200 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84014230 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) need {<0x1000, 0x1000> : [0xd000:1000~400]} 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xd000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1600~200 = 512 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1a00~200 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1a00~200 = 512 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1e00~200 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1e00~200 = 512 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2200~200 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84014230 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) need 0x{<0x1000, 0x1000> : [0xd000:1000~400]} 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e817e9c88 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81a47b90 space 0x558e817e9c88 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xd000~400 = 1024 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xd800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014230 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xd800~400 = 1024 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xe000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xe000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xe800~400 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xe800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xf000~400 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8194e1d8 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xf000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e81a47ef0 space 0x558e8194e1d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xf800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0xf800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2200~200 = 512 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x10000~400 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2600~200 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x10000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x10800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x10800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x11000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2600~200 = 512 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x11000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2a00~200 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x11800~400 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x11800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x12000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2a00~200 = 512 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x12000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2e00~200 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x12800~400 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x12800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x13000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2e00~200 = 512 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x13000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3200~200 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x13800~400 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x13800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x14000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x14000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x14800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x14800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x15000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x15000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x15800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x15800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x16000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x16000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x16800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x16800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x17000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x17000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x17800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x17800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x18000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x18000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x18800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x18800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x19000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x19000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x19800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x19800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1a000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1a000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1a800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1a800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1b000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1b000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1b800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1b800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1c000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1c000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1c800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1c800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1d000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1d000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1d800~400 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1d800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1e000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1e000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8194e1d8 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1e800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584000 space 0x558e8194e1d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1e800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1f000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3200~200 = 512 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1f000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3600~200 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1f800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3600~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x1f800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x20000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x20000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x20800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3600~200 = 512 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x20800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3a00~200 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x21000~400 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x21000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x21800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3a00~200 = 512 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x21800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3e00~200 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x22000~400 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x22000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x22800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3e00~200 = 512 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x22800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x4200~200 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x23000~400 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4200~200 size 0x6000 (24576) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x23000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x23800~400 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x23800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) need {<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x24000~400 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x24000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x24800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x24800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x25000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x25000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x25800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x25800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x26000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x26000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x26800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x26800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x27000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x27000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x27800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x27800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x28000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x28000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x28800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x28800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x29000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x29000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x29800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x29800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2a000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2a000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2a800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2a800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2b000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2b000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2b800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2b800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2c000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2c000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2c800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2c800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2d000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2d000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2d800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2d800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2e000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2e000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2e800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2e800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2f000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2f000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2f800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x2f800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x30000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x30000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x30800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x30800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x31000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x31000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x31800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x31800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x32000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x32000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x32800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x32800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x33000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x33000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x33800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x33800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x34000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x34000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x34800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x34800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x35000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x35000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x35800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x35800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x36000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x36000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x36800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x36800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x37000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x37000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x37800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x37800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x38000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x38000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x38800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x38800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x39000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x39000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x39800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x39800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3a000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3a000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3a800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3a800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3b000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3b000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3b800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3b800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3c000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3c000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3c800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3c800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3d000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3d000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3d800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3d800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3e000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3e000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3e800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3e800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3f000~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f000~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3f000~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3f800~400 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f800~400 size 0xe000 (57344) 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 0x3f800~400 = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=28672 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:891f0d13:::smithi13140796-14:4 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:891f0d13:::smithi13140796-14:4 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 57344 2019-12-07T19:34:04.783+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# hinfo_key 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# hinfo_key = 0 2019-12-07T19:34:04.783+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.1s0 27/26 ECSubReadReply(tid=76, attrs_read=0)) v2 -- 0x558e8417d180 con 0x558e818d7000 2019-12-07T19:34:04.783+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e84177760 finish 2019-12-07T19:34:04.783+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e81867978 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584090 space 0x558e81867978 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.784+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x4200~200 = 512 2019-12-07T19:34:04.784+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x4600~200 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4600~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.784+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x4600~200 = 512 2019-12-07T19:34:04.784+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x4a00~200 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.784+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x4a00~200 = 512 2019-12-07T19:34:04.784+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x4e00~200 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.784+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x4e00~200 = 512 2019-12-07T19:34:04.784+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x5200~200 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5200~200 size 0x6000 (24576) 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) need {<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x5200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.784+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e81867978 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584120 space 0x558e81867978 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x5200~200 = 512 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x5600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x5600~200 = 512 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x5a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x5a00~200 = 512 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x5e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x5e00~200 = 512 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x6200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x6200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x6600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x6600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x6a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x6a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x6e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x6e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x7200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x7200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x7600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x7600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x7a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x7a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x7e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x7e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x8200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x8200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x8600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x8600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x8a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x8a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x8e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x8e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x9200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x9200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x9600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x9600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x9a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x9a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x9e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x9e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xa200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xa200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xa600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xa600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xaa00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xaa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xaa00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xae00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xae00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xae00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xb200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xb200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xb600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xb600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xba00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xba00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xba00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xbe00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xbe00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xbe00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xc200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xc200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xc600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xc600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xca00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xca00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xca00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xce00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xce00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xce00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xd200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xd200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xd600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xd600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xda00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xda00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xda00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xde00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xde00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xde00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xe200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xe200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xe600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xe600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xea00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xea00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xea00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xee00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xee00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xee00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xf200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xf200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xf600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xf600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xfa00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xfa00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xfe00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfe00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0xfe00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x10200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x10200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x10600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x10600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x10a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x10a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x10e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x10e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x11200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x11200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x11600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x11600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x11a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x11a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x11e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x11e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x12200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x12200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x12600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x12600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x12a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x12a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x12e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x12e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x13200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x13200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x13600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x13600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x13a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x13a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x13e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x13e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x14200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x14200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x14600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x14600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x14a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x14a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x14e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x14e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x15200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x15200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x15600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x15600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x15a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x15a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x15e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x15e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x16200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x16200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x16600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x16600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x16a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x16a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x16e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x16e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x17200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x17200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x17600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x17600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x17a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x17a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x17e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x17e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x18200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x18200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x18600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x18600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x18a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x18a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x18e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x18e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x19200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x19200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x19600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x19600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x19a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x19a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x19e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x19e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1a200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1a200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1a600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1a600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1aa00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1aa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1aa00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1ae00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ae00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1ae00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1b200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1b200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1b600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1b600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1ba00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ba00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1ba00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1be00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1be00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1be00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1c200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1c200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1c600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1c600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1ca00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ca00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1ca00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1ce00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ce00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1ce00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1d200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1d200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1d600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1d600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1da00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1da00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1da00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1de00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1de00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1de00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1e200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1e200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1e600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1e600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1ea00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ea00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1ea00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1ee00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ee00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1ee00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1f200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1f200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1f600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1f600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1fa00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1fa00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1fe00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fe00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x1fe00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x20200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x20200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x20600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x20600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x20a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x20a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x20e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x20e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x21200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x21200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x21600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x21600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x21a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x21a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x21e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x21e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x22200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x22200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x22600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x22600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x22a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x22a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x22e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x22e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x23200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x23200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x23600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x23600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x23a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x23a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x23e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x23e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x24200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x24200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x24600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x24600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x24a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x24a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x24e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x24e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x25200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x25200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x25600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x25600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x25a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x25a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x25e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x25e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x26200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x26200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x26600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x26600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x26a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x26a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x26e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x26e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x27200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x27200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x27600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x27600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x27a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x27a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x27e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x27e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x28200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x28200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x28600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x28600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x28a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x28a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x28e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x28e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x29200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x29200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x29600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x29600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x29a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x29a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x29e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x29e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2a200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2a200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2a600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2a600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2aa00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2aa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2aa00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2ae00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ae00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2ae00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2b200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2b200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2b600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2b600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2ba00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ba00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2ba00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2be00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2be00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2be00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2c200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2c200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2c600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2c600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2ca00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ca00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2ca00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2ce00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ce00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2ce00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2d200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2d200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2d600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2d600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2da00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2da00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2da00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2de00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2de00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2de00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2e200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2e200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2e600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2e600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2ea00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ea00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2ea00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2ee00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ee00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2ee00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2f200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2f200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2f600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2f600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2fa00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2fa00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2fe00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fe00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x2fe00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x30200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x30200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x30600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x30600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x30a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x30a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x30e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x30e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x31200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x31200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x31600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x31600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x31a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x31a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x31e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x31e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x32200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x32200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x32600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x32600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x32a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x32a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x32e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x32e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x33200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x33200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x33600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x33600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x33a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x33a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x33e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x33e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x34200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x34200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x34600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x34600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x34a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x34a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x34e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x34e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x35200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x35200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x35600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x35600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x35a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x35a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x35e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x35e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x36200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x36200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x36600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x36600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x36a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x36a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x36e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x36e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x37200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x37200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x37600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x37600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x37a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x37a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x37e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x37e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x38200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x38200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x38600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x38600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x38a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x38a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x38e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x38e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x39200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x39200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x39600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x39600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x39a00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39a00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x39a00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x39e00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39e00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x39e00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3a200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3a200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3a600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3a600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3aa00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3aa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3aa00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3ae00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ae00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3ae00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3b200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3b200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3b600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3b600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3ba00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ba00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3ba00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3be00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3be00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3be00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3c200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3c200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3c600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3c600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3ca00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ca00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3ca00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3ce00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ce00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3ce00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3d200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3d200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3d600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3d600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3da00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3da00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3da00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3de00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3de00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3de00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3e200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3e200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3e600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3e600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3ea00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ea00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3ea00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3ee00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ee00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3ee00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3f200~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f200~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3f200~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3f600~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f600~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3f600~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3fa00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fa00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3fa00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3fe00~200 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fe00~200 size 0x6000 (24576) 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x3fe00~200 = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=12288 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:9b8adabe:::smithi13140796-6:head 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:9b8adabe:::smithi13140796-6:head 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 24576 2019-12-07T19:34:04.785+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# hinfo_key 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# hinfo_key = 0 2019-12-07T19:34:04.785+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 27/26 ECSubReadReply(tid=119, attrs_read=0)) v2 -- 0x558e811dc280 con 0x558e818d7c00 2019-12-07T19:34:04.785+0000 7fe2c7265700 10 osd.0 27 dequeue_op 0x558e84177600 finish 2019-12-07T19:34:04.785+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.789+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 164 ==== MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=120, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (unknown 0 0 0) 0x558e811dc280 con 0x558e818d7c00 2019-12-07T19:34:04.789+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83b0a580 prio 3 cost 0 latency 0.000028 epoch 27 MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=120, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.789+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=120, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=120, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=120, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e8192b000 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e83b0a580 prio 3 cost 0 latency 0.000092 MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=120, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=120, to_read={1:9da56077:::smithi13140796-13:head=0,262144,0}, subchunks={1:9da56077:::smithi13140796-13:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.789+0000 7fe2cb26d700 25 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x200~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x200~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x600~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x600~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xa00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xa00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xe00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xe00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1200~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1200~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1600~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1600~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1a00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1a00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1e00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1e00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2200~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866e70 blob([0x51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e0e0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2200~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2600~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866e70 blob([0x51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e0e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2600~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2a00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866e70 blob([0x51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e0e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2a00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2e00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866e70 blob([0x51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e0e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2e00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3200~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866e70 blob([0x51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e0e0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3200~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3600~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866e70 blob([0x51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e0e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3600~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3a00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866e70 blob([0x51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e0e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3a00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3e00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866e70 blob([0x51000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e0e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3e00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x4200~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c642a0 blob([0x53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c641c0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x4200~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x4600~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c642a0 blob([0x53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c641c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x4600~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x4a00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c642a0 blob([0x53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c641c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x4a00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x4e00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c642a0 blob([0x53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c641c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x4e00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x5200~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c642a0 blob([0x53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c641c0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x5200~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x5600~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c642a0 blob([0x53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c641c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x5600~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x5a00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c642a0 blob([0x53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c641c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x5a00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x5e00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c642a0 blob([0x53000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c641c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x5e00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x6200~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64150 blob([0x55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65110 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x6200~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x6600~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64150 blob([0x55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65110 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x6600~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x6a00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64150 blob([0x55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65110 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x6a00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x6e00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64150 blob([0x55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65110 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x6e00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x7200~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64150 blob([0x55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65110 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x7200~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x7600~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64150 blob([0x55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65110 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x7600~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x7a00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64150 blob([0x55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65110 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x7a00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x7e00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64150 blob([0x55000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65110 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x7e00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x8200~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65180 blob([0x57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c651f0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x8200~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x8600~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65180 blob([0x57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c651f0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x8600~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x8a00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65180 blob([0x57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c651f0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x8a00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x8e00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65180 blob([0x57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c651f0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x8e00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x9200~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65180 blob([0x57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c651f0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x9200~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x9600~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65180 blob([0x57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c651f0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x9600~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x9a00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65180 blob([0x57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c651f0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x9a00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x9e00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65180 blob([0x57000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c651f0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x9e00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xa200~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65260 blob([0x59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c652d0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xa200~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xa600~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65260 blob([0x59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c652d0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xa600~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xaa00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xaa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65260 blob([0x59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c652d0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xaa00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xae00~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xae00~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65260 blob([0x59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c652d0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xae00~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xb200~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb200~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65260 blob([0x59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c652d0 sbid 0x0)) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.789+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xb200~200 = 512 2019-12-07T19:34:04.789+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xb600~200 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb600~200 size 0xd000 (53248) 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.789+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65260 blob([0x59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c652d0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xb600~200 = 512 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xba00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xba00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65260 blob([0x59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c652d0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xba00~200 = 512 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xbe00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xbe00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65260 blob([0x59000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c652d0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xbe00~200 = 512 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xc200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65340 blob([0x5b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83c653b0 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xc200~200 = 512 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xc600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65340 blob([0x5b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83c653b0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xc600~200 = 512 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xca00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xca00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65340 blob([0x5b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83c653b0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xca00~200 = 512 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xce00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xce00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65340 blob([0x5b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83c653b0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xce00~200 = 512 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xd200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xd200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xd600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xd600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xda00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xda00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xda00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xde00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xde00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xde00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xe200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xe200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xe600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xe600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xea00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xea00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xea00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xee00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xee00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xee00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xf200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xf200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xf600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xf600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xfa00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xfa00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xfe00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfe00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xfe00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x10200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x10200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x10600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x10600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x10a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x10a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x10e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x10e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x11200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x11200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x11600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x11600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x11a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x11a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x11e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x11e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x12200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x12200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x12600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x12600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x12a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x12a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x12e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x12e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x13200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x13200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x13600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x13600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x13a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x13a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x13e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x13e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x14200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x14200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x14600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x14600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x14a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x14a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x14e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x14e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x15200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x15200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x15600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x15600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x15a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x15a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x15e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x15e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x16200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x16200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x16600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x16600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x16a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x16a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x16e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x16e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x17200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x17200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x17600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x17600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x17a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x17a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x17e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x17e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x18200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x18200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x18600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x18600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x18a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x18a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x18e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x18e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x19200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x19200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x19600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x19600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x19a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x19a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x19e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x19e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1a200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1a200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1a600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1a600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1aa00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1aa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1aa00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1ae00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ae00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1ae00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1b200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1b200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1b600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1b600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1ba00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ba00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1ba00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1be00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1be00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1be00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1c200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1c200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1c600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1c600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1ca00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ca00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1ca00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1ce00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ce00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1ce00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1d200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1d200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1d600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1d600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1da00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1da00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1da00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1de00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1de00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1de00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1e200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1e200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1e600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1e600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1ea00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ea00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1ea00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1ee00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ee00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1ee00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1f200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1f200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1f600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1f600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1fa00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1fa00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1fe00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fe00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x1fe00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x20200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x20200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x20600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x20600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x20a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x20a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x20e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x20e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x21200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x21200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x21600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x21600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x21a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x21a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x21e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x21e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x22200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x22200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x22600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x22600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x22a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x22a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x22e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x22e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x23200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x23200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x23600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x23600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x23a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x23a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x23e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x23e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x24200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x24200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x24600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x24600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x24a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x24a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x24e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x24e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x25200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x25200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x25600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x25600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x25a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x25a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x25e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x25e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x26200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x26200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x26600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x26600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x26a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x26a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x26e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x26e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x27200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x27200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x27600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x27600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x27a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x27a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x27e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x27e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x28200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x28200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x28600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x28600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x28a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x28a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x28e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x28e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x29200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x29200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x29600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x29600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x29a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x29a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x29e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x29e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2a200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2a200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2a600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2a600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2aa00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2aa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2aa00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2ae00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ae00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2ae00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2b200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2b200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2b600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2b600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2ba00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ba00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2ba00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2be00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2be00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2be00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2c200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2c200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2c600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2c600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2ca00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ca00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2ca00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2ce00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ce00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2ce00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2d200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2d200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2d600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2d600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2da00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2da00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2da00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2de00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2de00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2de00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2e200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2e200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2e600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2e600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2ea00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ea00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2ea00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2ee00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ee00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2ee00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2f200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2f200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2f600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2f600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2fa00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2fa00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2fe00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fe00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x2fe00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x30200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x30200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x30600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x30600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x30a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x30a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x30e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x30e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x31200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x31200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x31600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x31600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x31a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x31a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x31e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x31e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x32200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x32200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x32600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x32600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x32a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x32a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x32e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x32e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x33200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x33200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x33600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x33600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x33a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x33a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x33e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x33e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x34200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x34200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x34600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x34600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x34a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x34a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x34e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x34e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x35200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x35200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x35600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x35600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x35a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x35a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x35e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x35e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x36200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x36200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x36600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x36600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x36a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x36a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x36e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x36e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x37200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x37200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x37600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x37600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x37a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x37a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x37e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x37e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x38200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x38200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x38600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x38600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x38a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x38a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x38e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x38e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x39200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x39200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x39600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x39600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x39a00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39a00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x39a00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x39e00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39e00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x39e00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3a200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3a200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3a600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3a600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3aa00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3aa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3aa00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3ae00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ae00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3ae00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3b200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3b200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3b600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3b600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3ba00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ba00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3ba00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3be00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3be00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3be00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3c200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3c200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3c600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3c600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3ca00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ca00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3ca00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3ce00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ce00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3ce00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3d200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3d200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3d600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3d600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3da00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3da00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3da00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3de00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3de00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3de00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3e200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3e200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3e600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3e600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3ea00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ea00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3ea00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3ee00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ee00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3ee00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3f200~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f200~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3f200~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3f600~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f600~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3f600~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3fa00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fa00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3fa00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3fe00~200 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fe00~200 size 0xd000 (53248) 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3fe00~200 = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=26624 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 53248 2019-12-07T19:34:04.790+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key = 0 2019-12-07T19:34:04.790+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 27/26 ECSubReadReply(tid=120, attrs_read=0)) v2 -- 0x558e811dc780 con 0x558e818d7c00 2019-12-07T19:34:04.790+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e83b0a580 finish 2019-12-07T19:34:04.790+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.790+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 92 ==== MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=77, to_read={1:891f0d13:::smithi13140796-14:head=0,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,2,4,2]}, attrs_to_read=)) v3 ==== 249+0+0 (unknown 0 0 0) 0x558e8417d180 con 0x558e818d7000 2019-12-07T19:34:04.790+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edcf20 prio 3 cost 0 latency 0.000011 epoch 27 MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=77, to_read={1:891f0d13:::smithi13140796-14:head=0,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,2,4,2]}, attrs_to_read=)) v3 2019-12-07T19:34:04.790+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=77, to_read={1:891f0d13:::smithi13140796-14:head=0,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,2,4,2]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=77, to_read={1:891f0d13:::smithi13140796-14:head=0,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,2,4,2]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=77, to_read={1:891f0d13:::smithi13140796-14:head=0,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,2,4,2]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e819e4000 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 osd.0 27 dequeue_op 0x558e83edcf20 prio 3 cost 0 latency 0.000348 MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=77, to_read={1:891f0d13:::smithi13140796-14:head=0,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,2,4,2]}, attrs_to_read=)) v3 pg pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=77, to_read={1:891f0d13:::smithi13140796-14:head=0,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,2,4,2]}, attrs_to_read=)) v3 2019-12-07T19:34:04.791+0000 7fe2c7265700 25 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x0~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65730 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) need 0x0~400 cache has 0x[0~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x0~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65730 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65730 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) need 0x1000~400 cache has 0x[1000~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65730 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e00 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) need 0x0~400 cache has 0x[0~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e00 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e00 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) need 0x1000~400 cache has 0x[1000~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e00 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65810 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) need 0x0~400 cache has 0x[0~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65810 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65810 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) need 0x1000~400 cache has 0x[1000~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65810 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9c00 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) need 0x0~400 cache has 0x[0~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9c00 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9c00 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) need 0x1000~400 cache has 0x[1000~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9c00 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x8000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9420 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) need 0x0~400 cache has 0x[0~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x8000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x8800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9420 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x8800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x9000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9420 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) need 0x1000~400 cache has 0x[1000~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x9000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x9800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9420 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x9800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xa000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e95e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) need 0x0~400 cache has 0x[0~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xa000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xa800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e95e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xa800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xb000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e95e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) need 0x1000~400 cache has 0x[1000~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xb000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xb800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e95e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xb800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xc000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8850 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) need 0x0~400 cache has 0x[0~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xc000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xc800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8850 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xc800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xd000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8850 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) need 0x1000~400 cache has 0x[1000~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xd000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xd800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8850 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xd800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xe000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e5b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fa40 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194e5b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fa40 sbid 0x0)) need {<0x0, 0x1000> : [0xe000:0~400]} 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194e5b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fa40 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xe000:0~400]} 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8194fa58 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e845841b0 space 0x558e8194fa58 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xe000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xe800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e5b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fa40 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xe800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xf000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e5b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fa40 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194e5b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fa40 sbid 0x0)) need {<0x1000, 0x1000> : [0xf000:1000~400]} 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xf000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194e5b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fa40 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xf000:1000~400]} 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8194fa58 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584240 space 0x558e8194fa58 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xf000~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xf800~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e5b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fa40 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.791+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0xf800~400 = 1024 2019-12-07T19:34:04.791+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x10000~400 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f880 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9490 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194f880 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9490 sbid 0x0)) need {<0x0, 0x1000> : [0x10000:0~400]} 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.791+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194f880 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9490 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x10000:0~400]} 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e817e94a8 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e845842d0 space 0x558e817e94a8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.792+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x10000~400 = 1024 2019-12-07T19:34:04.792+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x10800~400 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f880 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9490 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.792+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x10800~400 = 1024 2019-12-07T19:34:04.792+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x11000~400 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f880 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9490 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194f880 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9490 sbid 0x0)) need {<0x1000, 0x1000> : [0x11000:1000~400]} 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x11000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194f880 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9490 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x11000:1000~400]} 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e817e94a8 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584360 space 0x558e817e94a8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.792+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x11000~400 = 1024 2019-12-07T19:34:04.792+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x11800~400 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f880 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9490 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.792+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x11800~400 = 1024 2019-12-07T19:34:04.792+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x12000~400 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866bd0 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2f50 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81866bd0 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2f50 sbid 0x0)) need {<0x0, 0x1000> : [0x12000:0~400]} 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81866bd0 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2f50 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12000:0~400]} 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a2f68 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e845843f0 space 0x558e841a2f68 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.792+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x12000~400 = 1024 2019-12-07T19:34:04.792+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x12800~400 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866bd0 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2f50 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.792+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x12800~400 = 1024 2019-12-07T19:34:04.792+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x13000~400 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866bd0 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2f50 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81866bd0 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2f50 sbid 0x0)) need {<0x1000, 0x1000> : [0x13000:1000~400]} 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x13000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81866bd0 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2f50 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x13000:1000~400]} 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a2f68 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584480 space 0x558e841a2f68 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.792+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x13000~400 = 1024 2019-12-07T19:34:04.792+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x13800~400 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866bd0 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2f50 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.792+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x13800~400 = 1024 2019-12-07T19:34:04.792+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x14000~400 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a2fc0 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3030 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a2fc0 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3030 sbid 0x0)) need {<0x0, 0x1000> : [0x14000:0~400]} 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a2fc0 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3030 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x14000:0~400]} 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3048 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584510 space 0x558e841a3048 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.792+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x14000~400 = 1024 2019-12-07T19:34:04.792+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x14800~400 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a2fc0 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3030 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.792+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x14800~400 = 1024 2019-12-07T19:34:04.792+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x15000~400 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a2fc0 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3030 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a2fc0 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3030 sbid 0x0)) need {<0x1000, 0x1000> : [0x15000:1000~400]} 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x15000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a2fc0 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3030 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x15000:1000~400]} 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3048 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e845845a0 space 0x558e841a3048 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.792+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x15000~400 = 1024 2019-12-07T19:34:04.792+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x15800~400 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a2fc0 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3030 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.792+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x15800~400 = 1024 2019-12-07T19:34:04.792+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x16000~400 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a30a0 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3110 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a30a0 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3110 sbid 0x0)) need {<0x0, 0x1000> : [0x16000:0~400]} 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a30a0 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3110 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x16000:0~400]} 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3128 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584630 space 0x558e841a3128 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.792+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x16000~400 = 1024 2019-12-07T19:34:04.792+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x16800~400 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a30a0 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3110 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.792+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x16800~400 = 1024 2019-12-07T19:34:04.792+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x17000~400 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a30a0 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3110 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.792+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a30a0 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3110 sbid 0x0)) need {<0x1000, 0x1000> : [0x17000:1000~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x17000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a30a0 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3110 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x17000:1000~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3128 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e845846c0 space 0x558e841a3128 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.793+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x17000~400 = 1024 2019-12-07T19:34:04.793+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x17800~400 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a30a0 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3110 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.793+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x17800~400 = 1024 2019-12-07T19:34:04.793+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x18000~400 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3180 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a31f0 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3180 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a31f0 sbid 0x0)) need {<0x0, 0x1000> : [0x18000:0~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3180 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a31f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18000:0~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3208 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584750 space 0x558e841a3208 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.793+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x18000~400 = 1024 2019-12-07T19:34:04.793+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x18800~400 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3180 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a31f0 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.793+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x18800~400 = 1024 2019-12-07T19:34:04.793+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x19000~400 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3180 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a31f0 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3180 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a31f0 sbid 0x0)) need {<0x1000, 0x1000> : [0x19000:1000~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x19000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3180 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a31f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x19000:1000~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3208 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e845847e0 space 0x558e841a3208 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.793+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x19000~400 = 1024 2019-12-07T19:34:04.793+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x19800~400 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3180 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a31f0 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.793+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x19800~400 = 1024 2019-12-07T19:34:04.793+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1a000~400 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3260 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a32d0 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3260 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a32d0 sbid 0x0)) need {<0x0, 0x1000> : [0x1a000:0~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3260 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a32d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1a000:0~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a32e8 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584870 space 0x558e841a32e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.793+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1a000~400 = 1024 2019-12-07T19:34:04.793+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1a800~400 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3260 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a32d0 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.793+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1a800~400 = 1024 2019-12-07T19:34:04.793+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1b000~400 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3260 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a32d0 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3260 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a32d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1b000:1000~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1b000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3260 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a32d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1b000:1000~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a32e8 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584900 space 0x558e841a32e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.793+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1b000~400 = 1024 2019-12-07T19:34:04.793+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1b800~400 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3260 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a32d0 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.793+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1b800~400 = 1024 2019-12-07T19:34:04.793+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1c000~400 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3340 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a33b0 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3340 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a33b0 sbid 0x0)) need {<0x0, 0x1000> : [0x1c000:0~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3340 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a33b0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1c000:0~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a33c8 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584990 space 0x558e841a33c8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.793+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1c000~400 = 1024 2019-12-07T19:34:04.793+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1c800~400 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3340 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a33b0 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.793+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1c800~400 = 1024 2019-12-07T19:34:04.793+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1d000~400 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3340 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a33b0 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3340 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a33b0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1d000:1000~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1d000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3340 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a33b0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1d000:1000~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a33c8 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584a20 space 0x558e841a33c8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.793+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1d000~400 = 1024 2019-12-07T19:34:04.793+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1d800~400 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3340 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a33b0 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.793+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1d800~400 = 1024 2019-12-07T19:34:04.793+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1e000~400 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3420 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3490 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3420 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3490 sbid 0x0)) need {<0x0, 0x1000> : [0x1e000:0~400]} 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.793+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3420 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3490 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1e000:0~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a34a8 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584ab0 space 0x558e841a34a8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1e000~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1e800~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3420 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3490 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1e800~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1f000~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3420 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3490 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3420 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3490 sbid 0x0)) need {<0x1000, 0x1000> : [0x1f000:1000~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1f000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3420 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3490 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1f000:1000~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a34a8 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584b40 space 0x558e841a34a8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1f000~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1f800~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3420 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3490 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x1f800~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x20000~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3500 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3570 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3500 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3570 sbid 0x0)) need {<0x0, 0x1000> : [0x20000:0~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3500 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3570 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x20000:0~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3588 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584bd0 space 0x558e841a3588 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x20000~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x20800~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3500 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3570 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x20800~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x21000~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3500 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3570 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3500 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3570 sbid 0x0)) need {<0x1000, 0x1000> : [0x21000:1000~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x21000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3500 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3570 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x21000:1000~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3588 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584c60 space 0x558e841a3588 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x21000~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x21800~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3500 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3570 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x21800~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x22000~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a35e0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3650 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a35e0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3650 sbid 0x0)) need {<0x0, 0x1000> : [0x22000:0~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a35e0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3650 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x22000:0~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3668 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584cf0 space 0x558e841a3668 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x22000~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x22800~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a35e0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3650 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x22800~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x23000~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a35e0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3650 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a35e0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3650 sbid 0x0)) need {<0x1000, 0x1000> : [0x23000:1000~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x23000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a35e0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3650 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x23000:1000~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3668 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584d80 space 0x558e841a3668 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x23000~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x23800~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a35e0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3650 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x23800~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x24000~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a36c0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3730 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a36c0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3730 sbid 0x0)) need {<0x0, 0x1000> : [0x24000:0~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a36c0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3730 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x24000:0~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3748 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584e10 space 0x558e841a3748 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x24000~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x24800~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a36c0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3730 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x24800~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x25000~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a36c0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3730 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a36c0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3730 sbid 0x0)) need {<0x1000, 0x1000> : [0x25000:1000~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x25000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a36c0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3730 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x25000:1000~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3748 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584ea0 space 0x558e841a3748 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x25000~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x25800~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a36c0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3730 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x25800~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x26000~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a37a0 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3810 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a37a0 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3810 sbid 0x0)) need {<0x0, 0x1000> : [0x26000:0~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a37a0 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3810 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x26000:0~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3828 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584f30 space 0x558e841a3828 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x26000~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x26800~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a37a0 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3810 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x26800~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x27000~400 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a37a0 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3810 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a37a0 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3810 sbid 0x0)) need {<0x1000, 0x1000> : [0x27000:1000~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x27000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a37a0 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3810 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x27000:1000~400]} 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3828 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.794+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84584fc0 space 0x558e841a3828 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.794+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x27000~400 = 1024 2019-12-07T19:34:04.794+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x27800~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a37a0 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3810 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x27800~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x28000~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3880 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a38f0 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3880 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a38f0 sbid 0x0)) need {<0x0, 0x1000> : [0x28000:0~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3880 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a38f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x28000:0~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3908 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585050 space 0x558e841a3908 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x28000~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x28800~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3880 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a38f0 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x28800~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x29000~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3880 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a38f0 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3880 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a38f0 sbid 0x0)) need {<0x1000, 0x1000> : [0x29000:1000~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x29000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3880 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a38f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x29000:1000~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3908 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e845850e0 space 0x558e841a3908 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x29000~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x29800~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3880 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a38f0 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x29800~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2a000~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3960 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a39d0 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3960 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a39d0 sbid 0x0)) need {<0x0, 0x1000> : [0x2a000:0~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3960 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a39d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2a000:0~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a39e8 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585170 space 0x558e841a39e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2a000~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2a800~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3960 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a39d0 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2a800~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2b000~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3960 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a39d0 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3960 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a39d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x2b000:1000~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2b000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3960 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a39d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2b000:1000~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a39e8 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585200 space 0x558e841a39e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2b000~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2b800~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3960 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a39d0 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2b800~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2c000~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3a40 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3ab0 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3a40 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3ab0 sbid 0x0)) need {<0x0, 0x1000> : [0x2c000:0~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2c000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3a40 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3ab0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2c000:0~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3ac8 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585290 space 0x558e841a3ac8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2c000~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2c800~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3a40 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3ab0 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2c800~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2d000~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3a40 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3ab0 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3a40 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3ab0 sbid 0x0)) need {<0x1000, 0x1000> : [0x2d000:1000~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2d000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3a40 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3ab0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2d000:1000~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3ac8 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585320 space 0x558e841a3ac8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2d000~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2d800~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3a40 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3ab0 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2d800~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2e000~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3b20 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3b90 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3b20 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3b90 sbid 0x0)) need {<0x0, 0x1000> : [0x2e000:0~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2e000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3b20 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3b90 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2e000:0~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3ba8 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e845853b0 space 0x558e841a3ba8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2e000~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2e800~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3b20 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3b90 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2e800~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2f000~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3b20 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3b90 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3b20 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3b90 sbid 0x0)) need {<0x1000, 0x1000> : [0x2f000:1000~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2f000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3b20 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3b90 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2f000:1000~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3ba8 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585440 space 0x558e841a3ba8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2f000~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2f800~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3b20 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3b90 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x2f800~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x30000~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3c00 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3c70 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3c00 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3c70 sbid 0x0)) need {<0x0, 0x1000> : [0x30000:0~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x30000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3c00 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3c70 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x30000:0~400]} 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3c88 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e845854d0 space 0x558e841a3c88 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x30000~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x30800~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3c00 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3c70 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.795+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x30800~400 = 1024 2019-12-07T19:34:04.795+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x31000~400 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3c00 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3c70 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.795+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3c00 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3c70 sbid 0x0)) need {<0x1000, 0x1000> : [0x31000:1000~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x31000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3c00 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3c70 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x31000:1000~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3c88 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585560 space 0x558e841a3c88 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x31000~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x31800~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3c00 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3c70 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x31800~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x32000~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3ce0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3d50 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3ce0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3d50 sbid 0x0)) need {<0x0, 0x1000> : [0x32000:0~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x32000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3ce0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3d50 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x32000:0~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3d68 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e845855f0 space 0x558e841a3d68 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x32000~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x32800~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3ce0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3d50 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x32800~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x33000~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3ce0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3d50 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3ce0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3d50 sbid 0x0)) need {<0x1000, 0x1000> : [0x33000:1000~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x33000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3ce0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3d50 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x33000:1000~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3d68 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585680 space 0x558e841a3d68 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x33000~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x33800~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3ce0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3d50 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x33800~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x34000~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3dc0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3e30 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3dc0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3e30 sbid 0x0)) need {<0x0, 0x1000> : [0x34000:0~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x34000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3dc0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3e30 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x34000:0~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3e48 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585710 space 0x558e841a3e48 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x34000~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x34800~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3dc0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3e30 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x34800~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x35000~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3dc0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3e30 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a3dc0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3e30 sbid 0x0)) need {<0x1000, 0x1000> : [0x35000:1000~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x35000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a3dc0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3e30 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x35000:1000~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841a3e48 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e845857a0 space 0x558e841a3e48 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x35000~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x35800~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3dc0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3e30 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x35800~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x36000~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03f80 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d5e0 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d03f80 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d5e0 sbid 0x0)) need {<0x0, 0x1000> : [0x36000:0~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x36000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d03f80 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d5e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x36000:0~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8423d5f8 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585830 space 0x558e8423d5f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x36000~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x36800~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03f80 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d5e0 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x36800~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x37000~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03f80 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d5e0 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d03f80 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d5e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x37000:1000~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x37000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d03f80 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d5e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x37000:1000~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8423d5f8 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e845858c0 space 0x558e8423d5f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x37000~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x37800~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03f80 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d5e0 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x37800~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x38000~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d570 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d500 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423d570 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d500 sbid 0x0)) need {<0x0, 0x1000> : [0x38000:0~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x38000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423d570 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d500 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x38000:0~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8423d518 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585950 space 0x558e8423d518 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x38000~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x38800~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d570 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d500 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x38800~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x39000~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d570 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d500 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423d570 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d500 sbid 0x0)) need {<0x1000, 0x1000> : [0x39000:1000~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x39000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423d570 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d500 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x39000:1000~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8423d518 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e845859e0 space 0x558e8423d518 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x39000~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x39800~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d570 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d500 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.796+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x39800~400 = 1024 2019-12-07T19:34:04.796+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3a000~400 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d490 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d420 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423d490 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d420 sbid 0x0)) need {<0x0, 0x1000> : [0x3a000:0~400]} 2019-12-07T19:34:04.796+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3a000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423d490 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d420 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3a000:0~400]} 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8423d438 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585a70 space 0x558e8423d438 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3a000~400 = 1024 2019-12-07T19:34:04.797+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3a800~400 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d490 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d420 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3a800~400 = 1024 2019-12-07T19:34:04.797+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3b000~400 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d490 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d420 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423d490 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d420 sbid 0x0)) need {<0x1000, 0x1000> : [0x3b000:1000~400]} 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3b000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423d490 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d420 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3b000:1000~400]} 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8423d438 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585b00 space 0x558e8423d438 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3b000~400 = 1024 2019-12-07T19:34:04.797+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3b800~400 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d490 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d420 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3b800~400 = 1024 2019-12-07T19:34:04.797+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3c000~400 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d3b0 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d340 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423d3b0 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d340 sbid 0x0)) need {<0x0, 0x1000> : [0x3c000:0~400]} 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3c000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423d3b0 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d340 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3c000:0~400]} 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8423d358 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585b90 space 0x558e8423d358 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3c000~400 = 1024 2019-12-07T19:34:04.797+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3c800~400 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d3b0 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d340 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3c800~400 = 1024 2019-12-07T19:34:04.797+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3d000~400 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d3b0 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d340 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423d3b0 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d340 sbid 0x0)) need {<0x1000, 0x1000> : [0x3d000:1000~400]} 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3d000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423d3b0 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d340 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3d000:1000~400]} 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8423d358 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585c20 space 0x558e8423d358 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3d000~400 = 1024 2019-12-07T19:34:04.797+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3d800~400 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d3b0 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d340 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3d800~400 = 1024 2019-12-07T19:34:04.797+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3e000~400 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d2d0 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d260 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423d2d0 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d260 sbid 0x0)) need {<0x0, 0x1000> : [0x3e000:0~400]} 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3e000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423d2d0 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d260 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3e000:0~400]} 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8423d278 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585cb0 space 0x558e8423d278 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3e000~400 = 1024 2019-12-07T19:34:04.797+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3e800~400 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d2d0 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d260 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3e800~400 = 1024 2019-12-07T19:34:04.797+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3f000~400 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d2d0 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d260 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423d2d0 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d260 sbid 0x0)) need {<0x1000, 0x1000> : [0x3f000:1000~400]} 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3f000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423d2d0 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d260 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3f000:1000~400]} 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8423d278 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585d40 space 0x558e8423d278 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3f000~400 = 1024 2019-12-07T19:34:04.797+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3f800~400 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d2d0 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d260 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x3f800~400 = 1024 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=1024 len=131072 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 315392 2019-12-07T19:34:04.797+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key = 0 2019-12-07T19:34:04.797+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.1s0 27/26 ECSubReadReply(tid=77, attrs_read=0)) v2 -- 0x558e83b09680 con 0x558e818d7000 2019-12-07T19:34:04.797+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 165 ==== MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=121, to_read={1:9694a0e5:::smithi13140796-3:4=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 263+0+0 (unknown 0 0 0) 0x558e811dc780 con 0x558e818d7c00 2019-12-07T19:34:04.797+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edc580 prio 3 cost 0 latency 0.000022 epoch 27 MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=121, to_read={1:9694a0e5:::smithi13140796-3:4=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.797+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=121, to_read={1:9694a0e5:::smithi13140796-3:4=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=121, to_read={1:9694a0e5:::smithi13140796-3:4=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=121, to_read={1:9694a0e5:::smithi13140796-3:4=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e8192b000 2019-12-07T19:34:04.797+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e83edc580 prio 3 cost 0 latency 0.000058 MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=121, to_read={1:9694a0e5:::smithi13140796-3:4=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.797+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=121, to_read={1:9694a0e5:::smithi13140796-3:4=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.797+0000 7fe2c7265700 10 osd.0 27 dequeue_op 0x558e83edcf20 finish 2019-12-07T19:34:04.797+0000 7fe2cb26d700 25 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.797+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.797+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x200~200 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03420 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.797+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x200~200 = 512 2019-12-07T19:34:04.797+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x600~200 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03420 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.797+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x600~200 = 512 2019-12-07T19:34:04.797+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xa00~200 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03420 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.797+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xa00~200 = 512 2019-12-07T19:34:04.797+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xe00~200 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03420 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.797+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xe00~200 = 512 2019-12-07T19:34:04.797+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1200~200 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03420 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1)))) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d03420 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1)))) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d03420 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1)))) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e817e8d38 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81a47c20 space 0x558e817e8d38 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.797+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1200~200 = 512 2019-12-07T19:34:04.797+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1600~200 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1600~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03420 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.797+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1600~200 = 512 2019-12-07T19:34:04.797+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1a00~200 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03420 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.797+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1a00~200 = 512 2019-12-07T19:34:04.797+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1e00~200 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03420 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.797+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1e00~200 = 512 2019-12-07T19:34:04.797+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2200~200 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2200~200 size 0x3000 (12288) 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03260 blob([0x39000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=1)))) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d03260 blob([0x39000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=1)))) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.797+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d03260 blob([0x39000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=1)))) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8194f3c8 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81a47cb0 space 0x558e8194f3c8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2200~200 = 512 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03260 blob([0x39000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=1)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2600~200 = 512 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03260 blob([0x39000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=1)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2a00~200 = 512 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03260 blob([0x39000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=1)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2e00~200 = 512 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x4200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x4200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x4600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x4600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x4a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x4a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x4e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x4e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x5200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x5200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x5600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x5600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x5a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x5a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x5e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x5e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x6200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x6200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x6600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x6600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x6a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x6a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x6e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x6e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x7200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x7200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x7600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x7600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x7a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x7a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x7e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x7e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x8200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x8200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x8600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x8600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x8a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x8a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x8e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x8e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x9200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x9200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x9600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x9600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x9a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x9a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x9e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x9e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xa200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xa200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xa600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xa600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xaa00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xaa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xaa00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xae00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xae00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xae00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xb200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xb200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xb600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xb600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xba00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xba00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xba00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xbe00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xbe00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xbe00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xc200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xc200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xc600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xc600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xca00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xca00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xca00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xce00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xce00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xce00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xd200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xd200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xd600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xd600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xda00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xda00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xda00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xde00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xde00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xde00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xe200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xe200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xe600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xe600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xea00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xea00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xea00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xee00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xee00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xee00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xf200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xf200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xf600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xf600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xfa00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xfa00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xfe00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfe00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0xfe00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x10200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x10200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x10600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x10600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x10a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x10a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x10e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x10e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x11200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x11200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x11600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x11600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x11a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x11a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x11e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x11e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x12200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x12200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x12600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x12600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x12a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x12a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x12e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x12e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x13200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x13200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x13600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x13600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x13a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x13a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x13e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x13e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x14200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x14200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x14600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x14600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x14a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x14a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x14e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x14e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x15200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x15200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x15600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x15600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x15a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x15a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x15e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x15e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x16200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x16200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x16600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x16600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x16a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x16a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x16e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x16e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x17200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x17200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x17600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x17600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x17a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x17a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x17e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x17e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x18200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x18200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x18600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x18600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x18a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x18a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x18e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x18e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x19200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x19200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x19600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x19600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x19a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x19a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x19e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x19e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1a200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1a200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1a600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1a600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1aa00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1aa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1aa00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1ae00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ae00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1ae00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1b200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1b200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1b600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1b600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1ba00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ba00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1ba00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1be00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1be00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1be00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1c200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1c200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1c600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1c600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1ca00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ca00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1ca00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1ce00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ce00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1ce00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1d200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1d200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1d600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1d600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1da00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1da00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1da00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1de00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1de00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1de00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1e200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1e200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1e600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1e600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1ea00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ea00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1ea00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1ee00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ee00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1ee00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1f200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1f200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1f600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1f600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1fa00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1fa00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1fe00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fe00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x1fe00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x20200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x20200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x20600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x20600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x20a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x20a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x20e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x20e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x21200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x21200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x21600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x21600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x21a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x21a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x21e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x21e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x22200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x22200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x22600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x22600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x22a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x22a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x22e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x22e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x23200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x23200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x23600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x23600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x23a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x23a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x23e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x23e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x24200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x24200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x24600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x24600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x24a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x24a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x24e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x24e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x25200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x25200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x25600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x25600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x25a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x25a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x25e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x25e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x26200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x26200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x26600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x26600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x26a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x26a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x26e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x26e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x27200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x27200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x27600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x27600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x27a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x27a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x27e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x27e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x28200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x28200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x28600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x28600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x28a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x28a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x28e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x28e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x29200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x29200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x29600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x29600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x29a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x29a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x29e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x29e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2a200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2a200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2a600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2a600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2aa00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2aa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2aa00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2ae00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ae00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2ae00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2b200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2b200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2b600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2b600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2ba00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ba00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2ba00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2be00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2be00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2be00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2c200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2c200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2c600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2c600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2ca00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ca00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2ca00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2ce00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ce00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2ce00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2d200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2d200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2d600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2d600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2da00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2da00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2da00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2de00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2de00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2de00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2e200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2e200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2e600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2e600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2ea00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ea00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2ea00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2ee00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ee00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2ee00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2f200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2f200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2f600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2f600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2fa00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2fa00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2fe00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fe00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x2fe00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x30200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x30200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x30600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x30600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x30a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x30a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x30e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x30e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x31200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x31200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x31600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x31600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x31a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x31a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x31e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x31e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x32200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x32200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x32600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x32600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x32a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x32a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x32e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x32e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x33200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x33200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x33600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x33600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x33a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x33a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x33e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x33e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x34200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x34200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x34600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x34600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x34a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x34a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x34e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x34e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x35200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x35200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x35600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x35600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x35a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x35a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x35e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x35e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x36200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x36200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x36600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x36600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x36a00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x36a00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x36e00~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x36e00~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x37200~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37200~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x37200~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x37600~200 2019-12-07T19:34:04.798+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37600~200 size 0x3000 (12288) 2019-12-07T19:34:04.798+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x37600~200 = 0 2019-12-07T19:34:04.798+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x37a00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x37a00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x37e00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x37e00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x38200~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38200~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x38200~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x38600~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38600~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x38600~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x38a00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x38a00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x38e00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x38e00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x39200~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39200~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x39200~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x39600~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39600~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x39600~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x39a00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39a00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x39a00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x39e00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39e00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x39e00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3a200~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a200~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3a200~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3a600~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a600~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3a600~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3aa00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3aa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3aa00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3ae00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ae00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3ae00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3b200~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b200~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3b200~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3b600~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b600~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3b600~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3ba00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ba00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3ba00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3be00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3be00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3be00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3c200~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c200~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3c200~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3c600~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c600~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3c600~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3ca00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ca00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3ca00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3ce00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ce00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3ce00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3d200~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d200~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3d200~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3d600~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d600~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3d600~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3da00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3da00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3da00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3de00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3de00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3de00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3e200~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e200~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3e200~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3e600~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e600~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3e600~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3ea00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ea00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3ea00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3ee00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ee00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3ee00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3f200~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f200~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3f200~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3f600~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f600~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3f600~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3fa00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fa00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3fa00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3fe00~200 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fe00~200 size 0x3000 (12288) 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 0x3fe00~200 = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=6144 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:9694a0e5:::smithi13140796-3:4 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:9694a0e5:::smithi13140796-3:4 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 12288 2019-12-07T19:34:04.799+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# hinfo_key 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# hinfo_key = 0 2019-12-07T19:34:04.799+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 27/26 ECSubReadReply(tid=121, attrs_read=0)) v2 -- 0x558e811dc280 con 0x558e818d7c00 2019-12-07T19:34:04.799+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e83edc580 finish 2019-12-07T19:34:04.799+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.802+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 166 ==== MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=122, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 263+0+0 (unknown 0 0 0) 0x558e811dc280 con 0x558e818d7c00 2019-12-07T19:34:04.802+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edc6e0 prio 3 cost 0 latency 0.000023 epoch 27 MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=122, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.802+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=122, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=122, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=122, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e8192b000 2019-12-07T19:34:04.802+0000 7fe2c7265700 10 osd.0 27 dequeue_op 0x558e83edc6e0 prio 3 cost 0 latency 0.000067 MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=122, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.802+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.9s3 27/26 ECSubRead(tid=122, to_read={1:9694a0e5:::smithi13140796-3:head=0,262144,0}, subchunks={1:9694a0e5:::smithi13140796-3:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.802+0000 7fe2c7265700 25 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.802+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x200~200 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x200~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) need {<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x200:200~200]} 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83d03518 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585dd0 space 0x558e83d03518 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.802+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x200~200 = 512 2019-12-07T19:34:04.802+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x600~200 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x600~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.802+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x600~200 = 512 2019-12-07T19:34:04.802+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xa00~200 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.802+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xa00~200 = 512 2019-12-07T19:34:04.802+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xe00~200 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.802+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xe00~200 = 512 2019-12-07T19:34:04.802+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1200~200 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1200~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83d03518 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.802+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 93 ==== MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=78, to_read={1:891f0d13:::smithi13140796-14:head=262144,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,2,4,2]}, attrs_to_read=)) v3 ==== 249+0+0 (unknown 0 0 0) 0x558e83b09680 con 0x558e818d7000 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585e60 space 0x558e83d03518 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.802+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1200~200 = 512 2019-12-07T19:34:04.802+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1600~200 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1600~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edd4a0 prio 127 cost 0 latency 0.000021 epoch 27 MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=78, to_read={1:891f0d13:::smithi13140796-14:head=262144,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,2,4,2]}, attrs_to_read=)) v3 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.802+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=78, to_read={1:891f0d13:::smithi13140796-14:head=262144,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,2,4,2]}, attrs_to_read=)) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.802+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1600~200 = 512 2019-12-07T19:34:04.802+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1a00~200 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.802+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1a00~200 = 512 2019-12-07T19:34:04.802+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1e00~200 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d033b0 blob([0x312000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03500 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.802+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1e00~200 = 512 2019-12-07T19:34:04.802+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2200~200 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2200~200 size 0x24000 (147456) 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03570 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03570 sbid 0x0)) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.802+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.802+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.802+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=78, to_read={1:891f0d13:::smithi13140796-14:head=262144,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,2,4,2]}, attrs_to_read=)) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.802+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.802+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=78, to_read={1:891f0d13:::smithi13140796-14:head=262144,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,2,4,2]}, attrs_to_read=)) v3) prio 127 cost 0 e27) pg 0x558e819e4000 2019-12-07T19:34:04.802+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e83edd4a0 prio 127 cost 0 latency 0.000116 MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=78, to_read={1:891f0d13:::smithi13140796-14:head=262144,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,2,4,2]}, attrs_to_read=)) v3 pg pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.802+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.1s1 27/26 ECSubRead(tid=78, to_read={1:891f0d13:::smithi13140796-14:head=262144,262144,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,2,4,2]}, attrs_to_read=)) v3 2019-12-07T19:34:04.802+0000 7fe2cb26d700 25 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x40000~400 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x40000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d1f0 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d180 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423d1f0 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d180 sbid 0x0)) need {<0x0, 0x1000> : [0x40000:0~400]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x40000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03570 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83d03588 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84585ef0 space 0x558e83d03588 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2200~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2600~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2600~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03570 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2600~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2a00~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03570 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2a00~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2e00~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03570 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2e00~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3200~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3200~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03570 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03570 sbid 0x0)) need {<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423d1f0 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d180 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x40000:0~400]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423d198 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81a47d40 space 0x558e8423d198 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x40000~400 = 1024 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x40800~400 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x40800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d1f0 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d180 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x40800~400 = 1024 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x41000~400 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x41000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d1f0 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d180 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423d1f0 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d180 sbid 0x0)) need {<0x1000, 0x1000> : [0x41000:1000~400]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x41000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423d1f0 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d180 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x41000:1000~400]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423d198 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81a47dd0 space 0x558e8423d198 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x41000~400 = 1024 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x41800~400 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x41800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d1f0 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d180 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x41800~400 = 1024 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x42000~400 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x42000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d110 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d0a0 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423d110 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d0a0 sbid 0x0)) need {<0x0, 0x1000> : [0x42000:0~400]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x42000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03570 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423d110 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d0a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x42000:0~400]} 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83d03588 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423d0b8 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84032ab0 space 0x558e83d03588 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e81a47e60 space 0x558e8423d0b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3200~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3600~200 2019-12-07T19:34:04.803+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x42000~400 = 1024 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3600~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x42800~400 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x42800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03570 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3600~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3a00~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d110 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d0a0 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03570 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x42800~400 = 1024 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3a00~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3e00~200 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x43000~400 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x43000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d035e0 blob([0x314000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03570 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d110 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d0a0 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3e00~200 = 512 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423d110 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d0a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x43000:1000~400]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x43000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x4200~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4200~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03650 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03650 sbid 0x0)) need {<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423d110 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d0a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x43000:1000~400]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423d0b8 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e84250000 space 0x558e8423d0b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x43000~400 = 1024 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x43800~400 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03650 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x43800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83d03668 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e842506c0 space 0x558e83d03668 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d110 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d0a0 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x43800~400 = 1024 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x4200~200 = 512 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x44000~400 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x4600~200 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x44000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4600~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d030 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cfc0 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03650 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423d030 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cfc0 sbid 0x0)) need {<0x0, 0x1000> : [0x44000:0~400]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x44000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x4600~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x4a00~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03650 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x4a00~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x4e00~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03650 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x4e00~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x5200~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5200~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03650 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03650 sbid 0x0)) need {<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x5200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423d030 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cfc0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x44000:0~400]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423cfd8 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e84250090 space 0x558e8423cfd8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x44000~400 = 1024 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x44800~400 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x44800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d030 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cfc0 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x44800~400 = 1024 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x45000~400 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x45000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d030 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cfc0 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423d030 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cfc0 sbid 0x0)) need {<0x1000, 0x1000> : [0x45000:1000~400]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x45000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03650 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83d03668 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250750 space 0x558e83d03668 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423d030 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cfc0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x45000:1000~400]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423cfd8 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x5200~200 = 512 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e84250120 space 0x558e8423cfd8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x5600~200 2019-12-07T19:34:04.803+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x45000~400 = 1024 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5600~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x45800~400 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x45800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03650 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d030 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cfc0 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x5600~200 = 512 2019-12-07T19:34:04.803+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x45800~400 = 1024 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x5a00~200 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x46000~400 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x46000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03650 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423cf50 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cee0 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423cf50 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cee0 sbid 0x0)) need {<0x0, 0x1000> : [0x46000:0~400]} 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x5a00~200 = 512 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x46000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x5e00~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d036c0 blob([0x316000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03650 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x5e00~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x6200~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6200~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da000 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da000 sbid 0x0)) need {<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423cf50 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cee0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x46000:0~400]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423cef8 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e842501b0 space 0x558e8423cef8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x46000~400 = 1024 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x46800~400 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x46800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423cf50 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cee0 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x46800~400 = 1024 2019-12-07T19:34:04.803+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x47000~400 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x47000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423cf50 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cee0 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423cf50 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cee0 sbid 0x0)) need {<0x1000, 0x1000> : [0x47000:1000~400]} 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x47000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.803+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da000 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da018 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e842507e0 space 0x558e841da018 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x6200~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x6600~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6600~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da000 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x6600~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x6a00~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da000 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x6a00~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x6e00~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da000 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.803+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x6e00~200 = 512 2019-12-07T19:34:04.803+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x7200~200 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7200~200 size 0x24000 (147456) 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da000 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da000 sbid 0x0)) need {<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x7200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.803+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423cf50 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cee0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x47000:1000~400]} 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423cef8 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e84250240 space 0x558e8423cef8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x47000~400 = 1024 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x47800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x47800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423cf50 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cee0 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x47800~400 = 1024 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x48000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x48000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423ce70 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ce00 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423ce70 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ce00 sbid 0x0)) need {<0x0, 0x1000> : [0x48000:0~400]} 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x48000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da000 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da018 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250870 space 0x558e841da018 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x7200~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x7600~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7600~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da000 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x7600~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x7a00~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da000 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x7a00~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x7e00~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d037a0 blob([0x318000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da000 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x7e00~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x8200~200 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423ce70 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ce00 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x48000:0~400]} 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8200~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423ce18 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da0e0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e842502d0 space 0x558e8423ce18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da0e0 sbid 0x0)) need {<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x48000~400 = 1024 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x48800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x48800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423ce70 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ce00 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x48800~400 = 1024 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x49000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x49000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423ce70 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ce00 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8423ce70 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ce00 sbid 0x0)) need {<0x1000, 0x1000> : [0x49000:1000~400]} 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x49000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8423ce70 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ce00 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x49000:1000~400]} 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423ce18 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e84250360 space 0x558e8423ce18 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x49000~400 = 1024 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x49800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x49800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423ce70 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ce00 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x49800~400 = 1024 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4a000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4a000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a91570 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a91570 sbid 0x0)) need {<0x0, 0x1000> : [0x4a000:0~400]} 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4a000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da0e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da0f8 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250900 space 0x558e841da0f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x8200~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x8600~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8600~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da0e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x8600~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x8a00~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da0e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x8a00~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x8e00~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da0e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x8e00~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x9200~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9200~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da0e0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da0e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x9200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a91570 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4a000:0~400]} 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e81a91588 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e842503f0 space 0x558e81a91588 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4a000~400 = 1024 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4a800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4a800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a91570 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4a800~400 = 1024 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4b000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4b000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a91570 sbid 0x0)) need 0x1000~400 cache has 0x[] 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a91570 sbid 0x0)) need {<0x1000, 0x1000> : [0x4b000:1000~400]} 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4b000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da0e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da0f8 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250990 space 0x558e841da0f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x9200~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x9600~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9600~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da0e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x9600~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x9a00~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da0e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x9a00~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x9e00~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da070 blob([0x31a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da0e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x9e00~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xa200~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa200~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da1c0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da1c0 sbid 0x0)) need {<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a91570 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x4b000:1000~400]} 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e81a91588 in 0x558e80219500) _discard 0x1000~1000 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e84250480 space 0x558e81a91588 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4b000~400 = 1024 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4b800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4b800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a91570 sbid 0x0)) need 0x1800~400 cache has 0x[1800~400] 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4b800~400 = 1024 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4c000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4c000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a91500 blob([0x311000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8423d650 sbid 0x0)) need 0x0~400 cache has 0x[] 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a91500 blob([0x311000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8423d650 sbid 0x0)) need {<0x0, 0x1000> : [0x4c000:0~400]} 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4c000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da1c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da1d8 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250a20 space 0x558e841da1d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xa200~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xa600~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa600~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da1c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xa600~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xaa00~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xaa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da1c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xaa00~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xae00~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xae00~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da1c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.804+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xae00~200 = 512 2019-12-07T19:34:04.804+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xb200~200 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb200~200 size 0x24000 (147456) 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da1c0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da1c0 sbid 0x0)) need {<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xb200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.804+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a91500 blob([0x311000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8423d650 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4c000:0~400]} 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore.BufferSpace(0x558e8423d668 in 0x558e80219500) _discard 0x0~1000 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 _add level 1 near 0 on buffer(0x558e84250510 space 0x558e8423d668 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4c000~400 = 1024 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4c800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4c800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a91500 blob([0x311000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8423d650 sbid 0x0)) need 0x800~400 cache has 0x[800~400] 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4c800~400 = 1024 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4d000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4d000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4d000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4d800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4d800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4d800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4e000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4e000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4e000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4e800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4e800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4e800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4f000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4f000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4f000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4f800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4f800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x4f800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x50000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x50000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x50000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x50800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x50800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x50800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x51000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x51000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x51000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x51800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x51800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x51800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x52000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x52000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x52000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x52800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x52800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x52800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x53000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x53000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x53000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x53800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x53800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x53800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x54000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x54000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x54000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x54800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x54800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x54800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x55000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x55000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x55000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x55800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x55800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x55800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x56000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x56000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x56000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x56800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x56800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x56800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x57000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x57000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x57000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x57800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x57800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x57800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x58000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x58000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x58000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x58800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x58800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x58800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x59000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x59000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x59000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x59800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x59800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x59800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5a000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5a000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5a000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5a800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5a800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5a800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5b000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5b000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5b000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5b800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5b800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5b800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5c000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5c000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5c000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5c800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5c800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5c800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5d000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5d000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5d000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5d800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5d800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5d800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5e000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5e000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5e000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5e800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5e800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5e800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5f000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5f000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5f000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5f800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5f800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x5f800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x60000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x60000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x60000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x60800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x60800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x60800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x61000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x61000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x61000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x61800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x61800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x61800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x62000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x62000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x62000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x62800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x62800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x62800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x63000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x63000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x63000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x63800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x63800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x63800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x64000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x64000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x64000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x64800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x64800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x64800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x65000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x65000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x65000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x65800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x65800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x65800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x66000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x66000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x66000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x66800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x66800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x66800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x67000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x67000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x67000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x67800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x67800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x67800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x68000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x68000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x68000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x68800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x68800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x68800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x69000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x69000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x69000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x69800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x69800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x69800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6a000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6a000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6a000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6a800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6a800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6a800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6b000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6b000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6b000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6b800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6b800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6b800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6c000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6c000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6c000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6c800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6c800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6c800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6d000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6d000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6d000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6d800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6d800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6d800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6e000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6e000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6e000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6e800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6e800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6e800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6f000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6f000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6f000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6f800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6f800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x6f800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x70000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x70000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x70000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x70800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x70800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x70800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x71000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x71000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x71000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x71800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x71800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x71800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x72000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x72000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x72000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x72800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x72800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x72800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x73000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x73000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x73000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x73800~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x73800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x73800~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x74000~400 2019-12-07T19:34:04.804+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x74000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.804+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x74000~400 = 0 2019-12-07T19:34:04.804+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x74800~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x74800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x74800~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x75000~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x75000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x75000~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x75800~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x75800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x75800~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x76000~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x76000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x76000~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x76800~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x76800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x76800~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x77000~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x77000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x77000~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x77800~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x77800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x77800~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x78000~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x78000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x78000~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x78800~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x78800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x78800~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x79000~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x79000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x79000~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x79800~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x79800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x79800~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7a000~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7a000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7a000~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7a800~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7a800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7a800~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7b000~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7b000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7b000~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7b800~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7b800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7b800~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7c000~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7c000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7c000~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7c800~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7c800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7c800~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7d000~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7d000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7d000~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7d800~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7d800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7d800~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7e000~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7e000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7e000~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7e800~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7e800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7e800~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7f000~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7f000~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7f000~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7f800~400 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7f800~400 size 0x4d000 (315392) 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x7f800~400 = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=26624 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 315392 2019-12-07T19:34:04.805+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key = 0 2019-12-07T19:34:04.805+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.1s0 27/26 ECSubReadReply(tid=78, attrs_read=0)) v2 -- 0x558e811dc780 con 0x558e818d7000 2019-12-07T19:34:04.805+0000 7fe2cb26d700 10 osd.0 27 dequeue_op 0x558e83edd4a0 finish 2019-12-07T19:34:04.805+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da1c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da1d8 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250ab0 space 0x558e841da1d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xb200~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xb600~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da1c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xb600~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xba00~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xba00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da1c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xba00~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xbe00~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xbe00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da150 blob([0x31c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da1c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xbe00~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xc200~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da2a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da2a0 sbid 0x0)) need {<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da2a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da2b8 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250b40 space 0x558e841da2b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xc200~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xc600~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da2a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xc600~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xca00~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xca00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da2a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xca00~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xce00~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xce00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da2a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xce00~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xd200~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da2a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da2a0 sbid 0x0)) need {<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xd200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da2a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da2b8 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250bd0 space 0x558e841da2b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xd200~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xd600~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da2a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xd600~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xda00~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xda00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da2a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xda00~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xde00~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xde00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da230 blob([0x31e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da2a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xde00~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xe200~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da380 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da380 sbid 0x0)) need {<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da380 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da398 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250c60 space 0x558e841da398 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xe200~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xe600~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da380 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xe600~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xea00~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xea00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da380 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xea00~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xee00~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xee00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da380 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xee00~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xf200~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da380 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da380 sbid 0x0)) need {<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xf200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da380 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da398 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250cf0 space 0x558e841da398 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xf200~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xf600~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da380 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xf600~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xfa00~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da380 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xfa00~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xfe00~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfe00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da310 blob([0x320000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da380 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0xfe00~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x10200~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da460 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da460 sbid 0x0)) need {<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da460 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da478 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250d80 space 0x558e841da478 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x10200~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x10600~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da460 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x10600~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x10a00~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da460 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x10a00~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x10e00~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da460 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x10e00~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x11200~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da460 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da460 sbid 0x0)) need {<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x11200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da460 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da478 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250e10 space 0x558e841da478 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x11200~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x11600~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11600~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da460 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x11600~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x11a00~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da460 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x11a00~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x11e00~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da3f0 blob([0x322000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da460 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.805+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x11e00~200 = 512 2019-12-07T19:34:04.805+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x12200~200 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12200~200 size 0x24000 (147456) 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da540 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da540 sbid 0x0)) need {<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x12200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.805+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.806+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 469486 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da540 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da558 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250ea0 space 0x558e841da558 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x12200~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x12600~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da540 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x12600~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x12a00~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da540 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x12a00~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x12e00~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da540 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x12e00~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x13200~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da540 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da540 sbid 0x0)) need {<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x13200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da540 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da558 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250f30 space 0x558e841da558 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x13200~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x13600~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da540 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x13600~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x13a00~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da540 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x13a00~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x13e00~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da4d0 blob([0x324000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da540 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x13e00~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x14200~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da620 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da620 sbid 0x0)) need {<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x14200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da620 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da638 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84250fc0 space 0x558e841da638 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x14200~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x14600~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da620 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x14600~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x14a00~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da620 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x14a00~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x14e00~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da620 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x14e00~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x15200~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da620 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da620 sbid 0x0)) need {<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x15200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da620 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da638 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84251050 space 0x558e841da638 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x15200~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x15600~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da620 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x15600~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x15a00~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da620 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x15a00~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x15e00~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da5b0 blob([0x326000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da620 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x15e00~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x16200~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da700 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da700 sbid 0x0)) need {<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x16200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da700 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da718 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e842510e0 space 0x558e841da718 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x16200~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x16600~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da700 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x16600~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x16a00~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da700 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x16a00~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x16e00~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da700 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x16e00~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x17200~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da700 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da700 sbid 0x0)) need {<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x17200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da700 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da718 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84251170 space 0x558e841da718 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x17200~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x17600~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da700 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x17600~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x17a00~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da700 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x17a00~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x17e00~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da690 blob([0x328000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da700 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x17e00~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x18200~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da7e0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da7e0 sbid 0x0)) need {<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x18200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da7e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da7f8 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84251200 space 0x558e841da7f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x18200~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x18600~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18600~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da7e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x18600~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x18a00~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da7e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x18a00~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x18e00~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da7e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.806+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x18e00~200 = 512 2019-12-07T19:34:04.806+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x19200~200 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19200~200 size 0x24000 (147456) 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da7e0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da7e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x19200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.806+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da7e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da7f8 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84251290 space 0x558e841da7f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x19200~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x19600~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19600~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da7e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x19600~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x19a00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da7e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x19a00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x19e00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da770 blob([0x32a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da7e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x19e00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1a200~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a200~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da8c0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da8c0 sbid 0x0)) need {<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1a200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da8c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da8d8 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84251320 space 0x558e841da8d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1a200~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1a600~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a600~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da8c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1a600~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1aa00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1aa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da8c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1aa00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1ae00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ae00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da8c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1ae00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1b200~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b200~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da8c0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da8c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1b200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da8c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da8d8 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e842513b0 space 0x558e841da8d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1b200~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1b600~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b600~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da8c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1b600~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1ba00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ba00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da8c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1ba00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1be00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1be00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da850 blob([0x32c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da8c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1be00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1c200~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c200~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da9a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da9a0 sbid 0x0)) need {<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1c200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da9a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da9b8 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84251440 space 0x558e841da9b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1c200~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1c600~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c600~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da9a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1c600~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1ca00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ca00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da9a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1ca00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1ce00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ce00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da9a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1ce00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1d200~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d200~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da9a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da9a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1d200:1200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1d200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da9a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1d200:1200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841da9b8 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e842514d0 space 0x558e841da9b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1d200~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1d600~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d600~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da9a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1d600~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1da00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1da00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da9a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1da00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1de00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1de00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841da930 blob([0x32e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841da9a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1de00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1e200~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e200~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841daa80 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841daa80 sbid 0x0)) need {<0x0, 0x1000> : [0x1e200:200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1e200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841daa80 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1e200:200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841daa98 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84251560 space 0x558e841daa98 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1e200~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1e600~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e600~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841daa80 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1e600~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1ea00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ea00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841daa80 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1ea00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1ee00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ee00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841daa80 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1ee00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1f200~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f200~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841daa80 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841daa80 sbid 0x0)) need {<0x1000, 0x1000> : [0x1f200:1200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1f200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841daa80 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1f200:1200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841daa98 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e842515f0 space 0x558e841daa98 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1f200~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1f600~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f600~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841daa80 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1f600~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1fa00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841daa80 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1fa00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1fe00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fe00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daa10 blob([0x330000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841daa80 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x1fe00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x20200~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20200~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dab60 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dab60 sbid 0x0)) need {<0x0, 0x1000> : [0x20200:200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x20200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dab60 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x20200:200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841dab78 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84251680 space 0x558e841dab78 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x20200~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x20600~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20600~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dab60 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x20600~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x20a00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dab60 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x20a00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x20e00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dab60 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x20e00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x21200~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21200~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dab60 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dab60 sbid 0x0)) need {<0x1000, 0x1000> : [0x21200:1200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x21200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dab60 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x21200:1200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841dab78 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84251710 space 0x558e841dab78 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x21200~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x21600~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21600~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dab60 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x21600~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x21a00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dab60 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x21a00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x21e00~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841daaf0 blob([0x332000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dab60 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x21e00~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x22200~200 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22200~200 size 0x24000 (147456) 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dac40 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dac40 sbid 0x0)) need {<0x0, 0x1000> : [0x22200:200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x22200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dac40 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x22200:200~200]} 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841dac58 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:04.807+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e842517a0 space 0x558e841dac58 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.807+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x22200~200 = 512 2019-12-07T19:34:04.807+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x22600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dac40 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x22600~200 = 512 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x22a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dac40 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x22a00~200 = 512 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x22e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dac40 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x22e00~200 = 512 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x23200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dac40 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dac40 sbid 0x0)) need {<0x1000, 0x1000> : [0x23200:1200~200]} 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x23200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dac40 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x23200:1200~200]} 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841dac58 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 _add level 1 near 0 on buffer(0x558e84251830 space 0x558e841dac58 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x23200~200 = 512 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x23600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dac40 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x23600~200 = 512 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x23a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dac40 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x23a00~200 = 512 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x23e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841dabd0 blob([0x334000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dac40 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x23e00~200 = 512 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x24200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x24200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x24600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x24600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x24a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x24a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x24e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x24e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x25200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x25200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x25600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x25600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x25a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x25a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x25e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x25e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x26200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x26200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x26600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x26600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x26a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x26a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x26e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x26e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x27200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x27200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x27600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x27600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x27a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x27a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x27e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x27e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x28200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x28200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x28600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x28600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x28a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x28a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x28e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x28e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x29200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x29200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x29600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x29600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x29a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x29a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x29e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x29e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2a200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2a200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2a600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2a600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2aa00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2aa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2aa00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2ae00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ae00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2ae00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2b200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2b200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2b600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2b600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2ba00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ba00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2ba00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2be00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2be00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2be00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2c200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2c200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2c600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2c600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2ca00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ca00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2ca00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2ce00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ce00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2ce00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2d200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2d200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2d600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2d600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2da00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2da00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2da00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2de00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2de00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2de00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2e200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2e200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2e600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2e600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2ea00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ea00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2ea00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2ee00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ee00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2ee00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2f200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2f200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2f600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2f600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2fa00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2fa00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2fe00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fe00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x2fe00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x30200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x30200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x30600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x30600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x30a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x30a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x30e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x30e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x31200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x31200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x31600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x31600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x31a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x31a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x31e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x31e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x32200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x32200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x32600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x32600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x32a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x32a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x32e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x32e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x33200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x33200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x33600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x33600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x33a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x33a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x33e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x33e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x34200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x34200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x34600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x34600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x34a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x34a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x34e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x34e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x35200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x35200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x35600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x35600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x35a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x35a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x35e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x35e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x36200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x36200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x36600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x36600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x36a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x36a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x36e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x36e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x37200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x37200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x37600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x37600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x37a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x37a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x37e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x37e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x38200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x38200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x38600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x38600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x38a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x38a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x38e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x38e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x39200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x39200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x39600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x39600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x39a00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39a00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x39a00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x39e00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39e00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x39e00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3a200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3a200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3a600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3a600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3aa00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3aa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3aa00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3ae00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ae00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3ae00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3b200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3b200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3b600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3b600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3ba00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ba00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3ba00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3be00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3be00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3be00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3c200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3c200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3c600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3c600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3ca00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ca00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3ca00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3ce00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ce00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3ce00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3d200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3d200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3d600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3d600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3da00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3da00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3da00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3de00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3de00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3de00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3e200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3e200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3e600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3e600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3ea00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ea00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3ea00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3ee00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ee00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3ee00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3f200~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f200~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3f200~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3f600~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f600~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3f600~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3fa00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fa00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3fa00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3fe00~200 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fe00~200 size 0x24000 (147456) 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 0x3fe00~200 = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=73728 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:9694a0e5:::smithi13140796-3:head 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:9694a0e5:::smithi13140796-3:head 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 147456 2019-12-07T19:34:04.808+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# hinfo_key 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# hinfo_key = 0 2019-12-07T19:34:04.808+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 27/26 ECSubReadReply(tid=122, attrs_read=0)) v2 -- 0x558e8417d180 con 0x558e818d7c00 2019-12-07T19:34:04.808+0000 7fe2c7265700 10 osd.0 27 dequeue_op 0x558e83edc6e0 finish 2019-12-07T19:34:04.808+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.809+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 94 ==== MRecoveryReserve(1.1s1 RELEASE e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e8403e1e0 con 0x558e818d7000 2019-12-07T19:34:04.809+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.1s1 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.809+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.809+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.809+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.809+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.809+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558e819e4000 2019-12-07T19:34:04.809+0000 7fe2cb26d700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.809+0000 7fe2cb26d700 5 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.810025+0000 1 0.000042 2019-12-07T19:34:04.809+0000 7fe2cb26d700 5 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.809+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:04.809+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.809+0000 7fe2cea74700 15 osd.0 27 enqueue_peering_evt 1.8s5 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.809+0000 7fe2cea74700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.809+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.809+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.809+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.809+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e841a4000 2019-12-07T19:34:04.809+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=8 mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.809+0000 7fe2c7a66700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.8s0 GRANT e27/27) v3 -- 0x558e817b4f00 con 0x558e818d7c00 2019-12-07T19:34:04.809+0000 7fe2c7a66700 5 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=8 mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.810494+0000 1 0.000030 2019-12-07T19:34:04.809+0000 7fe2c7a66700 5 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=8 mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.809+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:04.809+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.811+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 167 ==== MRecoveryReserve(1.9s3 RELEASE e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e817b4f00 con 0x558e818d7c00 2019-12-07T19:34:04.812+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.9s3 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.812+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.812+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.812+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.812+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.812+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558e8192b000 2019-12-07T19:34:04.812+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.812+0000 7fe2c7265700 5 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.812754+0000 1 0.000045 2019-12-07T19:34:04.812+0000 7fe2c7265700 5 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.812+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:04.812+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:04.812+0000 7fe2cea74700 15 osd.0 27 enqueue_peering_evt 1.fs1 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.812+0000 7fe2cea74700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.812+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.812+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.812+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 168 ==== MRecoveryReserve(1.as3 REQUEST prio: 190 e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e817b4f00 con 0x558e818d7c00 2019-12-07T19:34:04.812+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.812+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.812+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.as3 epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.812+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) 2019-12-07T19:34:04.812+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.812+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.812+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.812+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.812+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190) prio 255 cost 10 e27) pg 0x558e8193b000 2019-12-07T19:34:04.812+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MRecoveryReserve(1.fs0 GRANT e27/27) v3 -- 0x558e817b41e0 con 0x558e818d7000 2019-12-07T19:34:04.812+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RequestRecoveryPrio: priority 190 2019-12-07T19:34:04.812+0000 7fe2c8267700 5 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.812851+0000 1 0.000032 2019-12-07T19:34:04.812+0000 7fe2c8267700 5 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.812+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:04.812+0000 7fe2c6a64700 5 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.812866+0000 3 0.000127 2019-12-07T19:34:04.812+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.812+0000 7fe2c6a64700 5 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.812+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:04.812+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.815+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 95 ==== MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=79, to_read={1:f9bf9ba5:::smithi13140796-26:head=0,262144,0}, subchunks={1:f9bf9ba5:::smithi13140796-26:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (unknown 0 0 0) 0x558e8417d180 con 0x558e818d7000 2019-12-07T19:34:04.815+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edd760 prio 3 cost 0 latency 0.000029 epoch 27 MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=79, to_read={1:f9bf9ba5:::smithi13140796-26:head=0,262144,0}, subchunks={1:f9bf9ba5:::smithi13140796-26:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.815+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=79, to_read={1:f9bf9ba5:::smithi13140796-26:head=0,262144,0}, subchunks={1:f9bf9ba5:::smithi13140796-26:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=79, to_read={1:f9bf9ba5:::smithi13140796-26:head=0,262144,0}, subchunks={1:f9bf9ba5:::smithi13140796-26:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=79, to_read={1:f9bf9ba5:::smithi13140796-26:head=0,262144,0}, subchunks={1:f9bf9ba5:::smithi13140796-26:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e83edd760 prio 3 cost 0 latency 0.000075 MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=79, to_read={1:f9bf9ba5:::smithi13140796-26:head=0,262144,0}, subchunks={1:f9bf9ba5:::smithi13140796-26:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=79, to_read={1:f9bf9ba5:::smithi13140796-26:head=0,262144,0}, subchunks={1:f9bf9ba5:::smithi13140796-26:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.815+0000 7fe2c425f700 25 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x200~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ec40 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x200~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x600~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ec40 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x600~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xa00~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ec40 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xa00~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xe00~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ec40 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xe00~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1200~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ec40 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ec40 sbid 0x0)) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ec40 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e8194ec58 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e841ecab0 space 0x558e8194ec58 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1200~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1600~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ec40 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1600~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1a00~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ec40 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1a00~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1e00~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f730 blob([0xc8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ec40 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1e00~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2200~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5810 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5810 sbid 0x0)) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5810 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5828 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e841ecb40 space 0x558e83ac5828 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2200~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2600~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5810 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2600~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2a00~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5810 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2a00~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2e00~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5810 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2e00~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3200~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5810 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5810 sbid 0x0)) need {<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5810 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5828 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e841ecbd0 space 0x558e83ac5828 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3200~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3600~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5810 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3600~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3a00~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5810 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3a00~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3e00~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d36c40 blob([0xca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5810 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3e00~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x4200~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac57a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac57a0 sbid 0x0)) need {<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac57a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac57b8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84251a70 space 0x558e83ac57b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x4200~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x4600~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac57a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x4600~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x4a00~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac57a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x4a00~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x4e00~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac57a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.815+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x4e00~200 = 512 2019-12-07T19:34:04.815+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x5200~200 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac57a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac57a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x5200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.815+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac57a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac57b8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84251b00 space 0x558e83ac57b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x5200~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x5600~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac57a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x5600~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x5a00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac57a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x5a00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x5e00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5730 blob([0xcc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac57a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x5e00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x6200~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac58f0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac58f0 sbid 0x0)) need {<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac58f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5908 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84251b90 space 0x558e83ac5908 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x6200~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x6600~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac58f0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x6600~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x6a00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac58f0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x6a00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x6e00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac58f0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x6e00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x7200~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac58f0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac58f0 sbid 0x0)) need {<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x7200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac58f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5908 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84251c20 space 0x558e83ac5908 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x7200~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x7600~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac58f0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x7600~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x7a00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac58f0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x7a00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x7e00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5880 blob([0xce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac58f0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x7e00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x8200~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5ce0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5ce0 sbid 0x0)) need {<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5ce0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5cf8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84251cb0 space 0x558e83ac5cf8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x8200~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x8600~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5ce0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x8600~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x8a00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5ce0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x8a00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x8e00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5ce0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x8e00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x9200~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5ce0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5ce0 sbid 0x0)) need {<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x9200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5ce0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5cf8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84251d40 space 0x558e83ac5cf8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x9200~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x9600~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5ce0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x9600~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x9a00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5ce0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x9a00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x9e00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5960 blob([0xd0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5ce0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x9e00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xa200~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5a40 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5a40 sbid 0x0)) need {<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5a40 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5a58 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84251dd0 space 0x558e83ac5a58 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xa200~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xa600~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5a40 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xa600~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xaa00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xaa00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5a40 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xaa00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xae00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xae00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5a40 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xae00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xb200~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5a40 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5a40 sbid 0x0)) need {<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xb200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5a40 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83ac5a58 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84251e60 space 0x558e83ac5a58 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xb200~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xb600~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5a40 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xb600~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xba00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xba00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5a40 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xba00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xbe00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xbe00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4770 blob([0xd2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5a40 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xbe00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xc200~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ed90 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ed90 sbid 0x0)) need {<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ed90 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e8194eda8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84251ef0 space 0x558e8194eda8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xc200~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xc600~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ed90 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xc600~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xca00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xca00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ed90 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xca00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xce00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xce00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ed90 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xce00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xd200~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ed90 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ed90 sbid 0x0)) need {<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xd200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ed90 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e8194eda8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84258000 space 0x558e8194eda8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xd200~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xd600~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ed90 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xd600~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xda00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xda00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ed90 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xda00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xde00~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xde00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f180 blob([0xd4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ed90 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.816+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xde00~200 = 512 2019-12-07T19:34:04.816+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xe200~200 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e91f0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e91f0 sbid 0x0)) need {<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.816+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e91f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e817e9208 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84258090 space 0x558e817e9208 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xe200~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xe600~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e91f0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xe600~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xea00~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xea00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e91f0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xea00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xee00~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xee00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e91f0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xee00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xf200~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e91f0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e91f0 sbid 0x0)) need {<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xf200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e91f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e817e9208 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84258120 space 0x558e817e9208 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xf200~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xf600~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e91f0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xf600~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xfa00~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfa00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e91f0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xfa00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xfe00~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfe00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8d90 blob([0xd6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e91f0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0xfe00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x10200~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37500 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37500 sbid 0x0)) need {<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.817+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 169 ==== MOSDPGPush(1.8s5 27/26 [PushOp(1:1353159e:::smithi13140796-11:head, version: 21'4, data_included: [0~45056], data_size: 45056, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:1353159e:::smithi13140796-11:head@21'4, size: 180224, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:180224, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 46118+0+0 (unknown 0 0 0) 0x558e81239800 con 0x558e818d7c00 2019-12-07T19:34:04.817+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83b0ab00 prio 127 cost 46056 latency 0.000031 epoch 27 MOSDPGPush(1.8s5 27/26 [PushOp(1:1353159e:::smithi13140796-11:head, version: 21'4, data_included: [0~45056], data_size: 45056, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:1353159e:::smithi13140796-11:head@21'4, size: 180224, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:180224, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37500 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.817+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:1353159e:::smithi13140796-11:head, version: 21'4, data_included: [0~45056], data_size: 45056, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:1353159e:::smithi13140796-11:head@21'4, size: 180224, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:180224, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 46056 e27) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d37518 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842581b0 space 0x558e83d37518 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x10200~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x10600~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37500 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x10600~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x10a00~200 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37500 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x10a00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x10e00~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37500 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x10e00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x11200~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:1353159e:::smithi13140796-11:head, version: 21'4, data_included: [0~45056], data_size: 45056, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:1353159e:::smithi13140796-11:head@21'4, size: 180224, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:180224, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 46056 e27) queued 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37500 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37500 sbid 0x0)) need {<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x11200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:1353159e:::smithi13140796-11:head, version: 21'4, data_included: [0~45056], data_size: 45056, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:1353159e:::smithi13140796-11:head@21'4, size: 180224, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:180224, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 46056 e27) pg 0x558e841a4000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e83b0ab00 prio 127 cost 46056 latency 0.000096 MOSDPGPush(1.8s5 27/26 [PushOp(1:1353159e:::smithi13140796-11:head, version: 21'4, data_included: [0~45056], data_size: 45056, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:1353159e:::smithi13140796-11:head@21'4, size: 180224, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:180224, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=8 mbc={} ps=[2~1]] 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37500 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d37518 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84258240 space 0x558e83d37518 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x11200~200 = 512 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=8 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.8s5 27/26 [PushOp(1:1353159e:::smithi13140796-11:head, version: 21'4, data_included: [0~45056], data_size: 45056, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:1353159e:::smithi13140796-11:head@21'4, size: 180224, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:180224, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x11600~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37500 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x11600~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x11a00~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37500 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x11a00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x11e00~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37490 blob([0xd8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37500 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x11e00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x12200~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=8 mbc={} ps=[2~1]] on_local_recover: 1:1353159e:::smithi13140796-11:head 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d375e0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d375e0 sbid 0x0)) need {<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x12200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'3 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=8 mbc={} ps=[2~1]] got missing 1:1353159e:::smithi13140796-11:head v 21'4 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'11 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 21'3 active m=7 mbc={} ps=[2~1]] last_complete now 21'11 log.complete_to 21'12 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d375e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d375f8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842582d0 space 0x558e83d375f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.817+0000 7fe2cba6e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x12200~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x12600~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d375e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x12600~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x12a00~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d375e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x12a00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x12e00~200 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e83e72900 seq 14 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d375e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x12e00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x13200~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d375e0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:1353159e:::smithi13140796-11:head# key 0x8580000000000000011353159e21736d'ithi13140796-11!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d375e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x13200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 5#1:1353159e:::smithi13140796-11:head# 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:1353159e:::smithi13140796-11:head# key 0x8580000000000000011353159e21736d'ithi13140796-11!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d375e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d375f8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84258360 space 0x558e83d375f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x13200~200 = 512 2019-12-07T19:34:04.817+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1137 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x13600~200 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# = 0 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 0x0~b000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#1:1353159e:::smithi13140796-11:head# 0x0~b000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d375e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x13600~200 = 512 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~b000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x13a00~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d375e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x13a00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x13e00~200 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72900 6 blobs 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37570 blob([0xda000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d375e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x13e00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x14200~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d376c0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d376c0 sbid 0x0)) need {<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x14200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d376c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d376d8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842583f0 space 0x558e83d376d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x14200~200 = 512 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x14600~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 fbmap_alloc 0x558e80233300 allocate 0xb000/1000,b000,0 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x484000~b000/1000,b000,0 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d376c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x484000~b000] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x14600~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x14a00~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac4b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a2af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d376c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x14a00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x14e00~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac4b60 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a2af0 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d376c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x14e00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x15200~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.blob(0x558e83ac4b60) get_ref 0x0~2000 Blob(0x558e83ac4b60 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a2af0 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.blob(0x558e83ac4b60) get_ref init 0x2000, 1000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d376c0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d376c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x15200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83ac4b60 blob([0x484000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2af0 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e841a2b08 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f49880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d371f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f49880 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d371f0 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f49880) get_ref 0x0~2000 Blob(0x558e83f49880 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d371f0 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f49880) get_ref init 0x2000, 1000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83f49880 blob([0x486000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d371f0 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83d37208 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423c230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f94d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423c230 blob([0x488000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f94d20 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.blob(0x558e8423c230) get_ref 0x0~2000 Blob(0x558e8423c230 blob([0x488000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f94d20 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.blob(0x558e8423c230) get_ref init 0x2000, 1000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d376c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e8423c230 blob([0x488000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f94d20 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d376d8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f94d38 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84258480 space 0x558e83d376d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x15200~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x15600~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b285b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b28000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d376c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b285b0 blob([0x48a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b28000 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x15600~200 = 512 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.blob(0x558e83b285b0) get_ref 0x0~2000 Blob(0x558e83b285b0 blob([0x48a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b28000 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.blob(0x558e83b285b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x15a00~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e83b285b0 blob([0x48a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28000 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83b28018 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d376c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x15a00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b28540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f61a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x15e00~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b28540 blob([0x48c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61a40 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.blob(0x558e83b28540) get_ref 0x0~2000 Blob(0x558e83b28540 blob([0x48c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61a40 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.blob(0x558e83b28540) get_ref init 0x2000, 1000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37650 blob([0xdc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d376c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e83b28540 blob([0x48c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f61a40 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x15e00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f61a58 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x16200~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d377a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d377a0 sbid 0x0)) need {<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x16200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f61ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f61b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f61ab0 blob([0x48e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61b20 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f61ab0) get_ref 0x0~1000 Blob(0x558e83f61ab0 blob([0x48e000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61b20 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f61ab0) get_ref init 0x1000, 1000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~1000: 0x0~1000 Blob(0x558e83f61ab0 blob([0x48e000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f61b20 sbid 0x0)) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f61b38 in 0x558e80219960) _discard 0x0~1000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0xb000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e84242108) dirty_range mark inline shard dirty 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 0x0~b000 = 0 2019-12-07T19:34:04.817+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 4 keys 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d377a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d377b8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:1353159e:::smithi13140796-11:head# 4 keys = 0 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84258510 space 0x558e83d377b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.817+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x16200~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x16600~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x000000000000045f'.missing/0000000000000001.8CAC8A97.head.smithi13140796-11..' <- missing/0000000000000001.8CAC8A97.head.smithi13140796-11.. 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d377a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.817+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x16600~200 = 512 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x16a00~200 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d377a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x16a00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x16e00~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72900 cost 75411 (7 ios * 4000 + 47411 bytes) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d377a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72900 onodes 0x558e84242000 shared_blobs 2019-12-07T19:34:04.817+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x16e00~200 = 512 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e84242108) update 5#1:1353159e:::smithi13140796-11:head# 2019-12-07T19:34:04.817+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x17200~200 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e84242108) update inline shard 114 bytes from 6 extents 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d377a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d377a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x17200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:1353159e:::smithi13140796-11:head# is 605 (485 bytes onode + 2 bytes spanning blobs + 118 bytes inline extents) 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72900 allocated 0x[484000~b000] released 0x[] 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 freelist allocate 0x484000~b000 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 freelist _xor first_key 0x480000 last_key 0x480000 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72900 prepare 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83e72900 2019-12-07T19:34:04.817+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.817+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e83b0ab00 finish 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'11 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 21'3 active m=7 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.817+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.817+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72900 aio_wait 2019-12-07T19:34:04.817+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72900 2019-12-07T19:34:04.817+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72900 io_done 2019-12-07T19:34:04.817+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.817+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.817+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.817+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d377a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d377b8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842585a0 space 0x558e83d377b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x17200~200 = 512 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x17600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d377a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x17600~200 = 512 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x17a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d377a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x17a00~200 = 512 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x17e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37730 blob([0xde000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d377a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x17e00~200 = 512 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x18200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37880 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.818+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2046 < min_flush_size 524288 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37880 sbid 0x0)) need {<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x18200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.818+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84242000 had 1 2019-12-07T19:34:04.818+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.818+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2069 < min_flush_size 524288 2019-12-07T19:34:04.818+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2069 < min_flush_size 524288 2019-12-07T19:34:04.818+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.818+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xd08a8~815 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.818+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xd08a8 0xd08a8~815 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.818+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.818+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa08a8 2019-12-07T19:34:04.818+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x8a8 2019-12-07T19:34:04.818+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.818+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xbd and padding block with 0xf43 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37880 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d37898 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84258630 space 0x558e83d37898 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x18200~200 = 512 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x18600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.818+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xd10bd 2019-12-07T19:34:04.818+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.818+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37880 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.818+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.818+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x18600~200 = 512 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x18a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.818+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000177457s (0.000057324s flush + 0.000120133s kv commit) 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37880 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.818+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x18a00~200 = 512 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x18e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37880 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x18e00~200 = 512 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x19200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37880 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37880 sbid 0x0)) need {<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x19200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.818+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.818+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e72900> 2019-12-07T19:34:04.818+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.818+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72900 kv_submitted 2019-12-07T19:34:04.818+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72900 2019-12-07T19:34:04.818+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72900 finishing 2019-12-07T19:34:04.818+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72900 onodes 0x558e84242000 2019-12-07T19:34:04.818+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.818+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b28018 in 0x558e80219960) _finish_write discard buffer(0x558e84258ab0 space 0x558e83b28018 0x0~2000 writing nocache) 2019-12-07T19:34:04.818+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d37208 in 0x558e80219960) _finish_write discard buffer(0x558e84258990 space 0x558e83d37208 0x0~2000 writing nocache) 2019-12-07T19:34:04.818+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f61a58 in 0x558e80219960) _finish_write discard buffer(0x558e84258b40 space 0x558e83f61a58 0x0~2000 writing nocache) 2019-12-07T19:34:04.818+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f61b38 in 0x558e80219960) _finish_write discard buffer(0x558e84258bd0 space 0x558e83f61b38 0x0~1000 writing nocache) 2019-12-07T19:34:04.818+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f94d38 in 0x558e80219960) _finish_write discard buffer(0x558e84258a20 space 0x558e83f94d38 0x0~2000 writing nocache) 2019-12-07T19:34:04.818+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a2b08 in 0x558e80219960) _finish_write discard buffer(0x558e84258900 space 0x558e841a2b08 0x0~2000 writing nocache) 2019-12-07T19:34:04.818+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72900 done 2019-12-07T19:34:04.818+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.818+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72900 [] 2019-12-07T19:34:04.818+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.818+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'11 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 21'3 active m=7 mbc={} ps=[2~1]] recovery_committed_to version 21'11 now ondisk 2019-12-07T19:34:04.818+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.818+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 27/26 [PushReplyOp(1:1353159e:::smithi13140796-11:head)]) v3 -- 0x558e818ee600 con 0x558e818d7c00 2019-12-07T19:34:04.818+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37880 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d37898 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842586c0 space 0x558e83d37898 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x19200~200 = 512 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x19600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37880 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x19600~200 = 512 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x19a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37880 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x19a00~200 = 512 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x19e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d37810 blob([0xe0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d37880 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x19e00~200 = 512 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1a200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1a200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1a600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1a600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1aa00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1aa00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1aa00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1ae00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ae00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1ae00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1b200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1b200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1b600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1b600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1ba00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ba00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1ba00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1be00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1be00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1be00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1c200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1c200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1c600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1c600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1ca00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ca00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1ca00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1ce00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ce00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1ce00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1d200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1d200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1d600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1d600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1da00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1da00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1da00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1de00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1de00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1de00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1e200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1e200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1e600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1e600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1ea00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ea00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1ea00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1ee00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ee00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1ee00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1f200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1f200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1f600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1f600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1fa00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fa00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1fa00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1fe00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fe00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x1fe00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x20200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x20200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x20600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x20600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x20a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x20a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x20e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x20e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x21200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x21200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x21600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x21600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x21a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x21a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x21e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x21e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x22200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x22200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x22600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x22600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x22a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x22a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x22e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x22e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x23200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x23200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x23600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x23600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x23a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x23a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x23e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x23e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x24200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x24200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x24600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x24600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x24a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x24a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x24e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x24e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x25200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x25200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x25600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x25600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x25a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x25a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x25e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x25e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x26200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x26200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x26600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x26600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x26a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x26a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x26e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x26e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x27200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x27200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x27600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x27600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x27a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x27a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x27e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x27e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x28200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x28200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x28600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x28600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x28a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x28a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x28e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x28e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x29200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x29200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x29600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x29600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x29a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x29a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x29e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x29e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2a200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2a200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2a600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2a600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2aa00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2aa00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2aa00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2ae00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ae00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2ae00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2b200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2b200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2b600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2b600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2ba00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ba00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2ba00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2be00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2be00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2be00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2c200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2c200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2c600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2c600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2ca00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ca00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2ca00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2ce00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ce00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2ce00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2d200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2d200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2d600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2d600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2da00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2da00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2da00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2de00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2de00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2de00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2e200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2e200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2e600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2e600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2ea00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ea00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2ea00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2ee00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ee00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2ee00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2f200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2f200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2f600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2f600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2fa00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fa00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2fa00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2fe00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fe00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x2fe00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x30200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x30200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x30600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x30600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x30a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x30a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x30e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x30e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x31200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x31200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x31600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x31600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x31a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x31a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x31e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x31e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x32200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x32200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x32600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x32600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x32a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x32a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x32e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x32e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x33200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x33200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x33600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x33600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x33a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x33a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x33e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x33e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x34200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x34200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x34600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x34600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x34a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x34a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x34e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x34e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x35200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x35200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x35600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x35600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x35a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x35a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x35e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x35e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x36200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x36200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x36600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x36600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x36a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x36a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x36e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x36e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x37200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x37200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x37600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x37600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x37a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x37a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x37e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x37e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x38200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x38200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x38600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x38600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x38a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x38a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x38e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x38e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x39200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x39200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x39600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x39600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x39a00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39a00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x39a00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x39e00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39e00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x39e00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3a200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3a200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3a600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3a600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3aa00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3aa00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3aa00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3ae00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ae00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3ae00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3b200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3b200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3b600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3b600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3ba00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ba00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3ba00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3be00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3be00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3be00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3c200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3c200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3c600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3c600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3ca00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ca00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3ca00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3ce00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ce00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3ce00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3d200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3d200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3d600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3d600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3da00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3da00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3da00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3de00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3de00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3de00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3e200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3e200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3e600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3e600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3ea00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ea00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3ea00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3ee00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ee00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3ee00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3f200~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f200~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3f200~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3f600~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f600~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3f600~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3fa00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fa00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3fa00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3fe00~200 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fe00~200 size 0x1a000 (106496) 2019-12-07T19:34:04.818+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 0x3fe00~200 = 0 2019-12-07T19:34:04.818+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=53248 2019-12-07T19:34:04.819+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:f9bf9ba5:::smithi13140796-26:head 2019-12-07T19:34:04.819+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:f9bf9ba5:::smithi13140796-26:head 2019-12-07T19:34:04.819+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# 2019-12-07T19:34:04.819+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 106496 2019-12-07T19:34:04.819+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# hinfo_key 2019-12-07T19:34:04.819+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:f9bf9ba5:::smithi13140796-26:head# hinfo_key = 0 2019-12-07T19:34:04.819+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.fs0 27/26 ECSubReadReply(tid=79, attrs_read=0)) v2 -- 0x558e811dc000 con 0x558e818d7000 2019-12-07T19:34:04.819+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e83edd760 finish 2019-12-07T19:34:04.819+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.825+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 96 ==== MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=80, to_read={1:fc3a86a2:::smithi13140796-33:head=0,262144,0}, subchunks={1:fc3a86a2:::smithi13140796-33:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (unknown 0 0 0) 0x558e811dc000 con 0x558e818d7000 2019-12-07T19:34:04.825+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83b0b340 prio 3 cost 0 latency 0.000031 epoch 27 MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=80, to_read={1:fc3a86a2:::smithi13140796-33:head=0,262144,0}, subchunks={1:fc3a86a2:::smithi13140796-33:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.825+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=80, to_read={1:fc3a86a2:::smithi13140796-33:head=0,262144,0}, subchunks={1:fc3a86a2:::smithi13140796-33:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=80, to_read={1:fc3a86a2:::smithi13140796-33:head=0,262144,0}, subchunks={1:fc3a86a2:::smithi13140796-33:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=80, to_read={1:fc3a86a2:::smithi13140796-33:head=0,262144,0}, subchunks={1:fc3a86a2:::smithi13140796-33:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 osd.0 27 dequeue_op 0x558e83b0b340 prio 3 cost 0 latency 0.000080 MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=80, to_read={1:fc3a86a2:::smithi13140796-33:head=0,262144,0}, subchunks={1:fc3a86a2:::smithi13140796-33:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=80, to_read={1:fc3a86a2:::smithi13140796-33:head=0,262144,0}, subchunks={1:fc3a86a2:::smithi13140796-33:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.825+0000 7fe2c8267700 25 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x200~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x200~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f960 sbid 0x0)) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x200~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x600~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x600~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f960 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x600~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xa00~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa00~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f960 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xa00~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xe00~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe00~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f960 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xe00~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1200~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1200~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f960 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f960 sbid 0x0)) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f960 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e8194f978 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84033dd0 space 0x558e8194f978 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1200~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1600~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1600~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f960 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1600~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1a00~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f960 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1a00~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1e00~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac5e30 blob([0x135000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f960 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1e00~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2200~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2200~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ebd0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ebd0 sbid 0x0)) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ebd0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0ebe8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e841ed9e0 space 0x558e83d0ebe8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2200~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2600~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2600~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ebd0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2600~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2a00~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ebd0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2a00~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2e00~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ebd0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2e00~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3200~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3200~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ebd0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ebd0 sbid 0x0)) need {<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ebd0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0ebe8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e841eda70 space 0x558e83d0ebe8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3200~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3600~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3600~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ebd0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3600~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3a00~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ebd0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3a00~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3e00~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4d20 blob([0x137000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ebd0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3e00~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x4200~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4200~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0eb60 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0eb60 sbid 0x0)) need {<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0eb60 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0eb78 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e841edb00 space 0x558e83d0eb78 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x4200~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x4600~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4600~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0eb60 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x4600~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x4a00~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0eb60 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x4a00~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x4e00~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0eb60 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x4e00~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x5200~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5200~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0eb60 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0eb60 sbid 0x0)) need {<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x5200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0eb60 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0eb78 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83d106c0 space 0x558e83d0eb78 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x5200~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x5600~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5600~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0eb60 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x5600~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x5a00~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0eb60 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x5a00~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x5e00~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9b20 blob([0x139000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0eb60 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x5e00~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x6200~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6200~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ecb0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ecb0 sbid 0x0)) need {<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ecb0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0ecc8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83d11950 space 0x558e83d0ecc8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x6200~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x6600~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6600~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ecb0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x6600~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x6a00~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ecb0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x6a00~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x6e00~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ecb0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.825+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x6e00~200 = 512 2019-12-07T19:34:04.825+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x7200~200 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7200~200 size 0x21000 (135168) 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ecb0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ecb0 sbid 0x0)) need {<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x7200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.825+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ecb0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0ecc8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e83c8f050 space 0x558e83d0ecc8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x7200~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x7600~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7600~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ecb0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x7600~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x7a00~200 2019-12-07T19:34:04.826+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 170 ==== MOSDPGPush(1.8s5 27/26 [PushOp(1:10f769a3:::smithi13140796-42:head, version: 21'12, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10f769a3:::smithi13140796-42:head@21'12, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 177190+0+0 (unknown 0 0 0) 0x558e818ee600 con 0x558e818d7c00 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ecb0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.826+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83b0adc0 prio 127 cost 177128 latency 0.000032 epoch 27 MOSDPGPush(1.8s5 27/26 [PushOp(1:10f769a3:::smithi13140796-42:head, version: 21'12, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10f769a3:::smithi13140796-42:head@21'12, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x7a00~200 = 512 2019-12-07T19:34:04.826+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:10f769a3:::smithi13140796-42:head, version: 21'12, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10f769a3:::smithi13140796-42:head@21'12, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 177128 e27) 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x7e00~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ec40 blob([0x13b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ecb0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x7e00~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x8200~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8200~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ed90 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ed90 sbid 0x0)) need {<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:10f769a3:::smithi13140796-42:head, version: 21'12, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10f769a3:::smithi13140796-42:head@21'12, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 177128 e27) queued 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:10f769a3:::smithi13140796-42:head, version: 21'12, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10f769a3:::smithi13140796-42:head@21'12, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 177128 e27) pg 0x558e841a4000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e83b0adc0 prio 127 cost 177128 latency 0.000072 MOSDPGPush(1.8s5 27/26 [PushOp(1:10f769a3:::smithi13140796-42:head, version: 21'12, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10f769a3:::smithi13140796-42:head@21'12, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 27'23 lc 21'11 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=7 mbc={} ps=[2~1]] 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'11 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=7 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.8s5 27/26 [PushOp(1:10f769a3:::smithi13140796-42:head, version: 21'12, data_included: [0~176128], data_size: 176128, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10f769a3:::smithi13140796-42:head@21'12, size: 704512, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:704512, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'11 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=7 mbc={} ps=[2~1]] on_local_recover: 1:10f769a3:::smithi13140796-42:head 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 21'11 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=7 mbc={} ps=[2~1]] got missing 1:10f769a3:::smithi13140796-42:head v 21'12 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 23'14 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 21'11 active m=6 mbc={} ps=[2~1]] last_complete now 23'14 log.complete_to 27'15 2019-12-07T19:34:04.826+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ed90 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e83e72000 seq 15 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0eda8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84258ea0 space 0x558e83d0eda8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x8200~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x8600~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8600~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:10f769a3:::smithi13140796-42:head# key 0x85800000000000000110f769a321736d'ithi13140796-42!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ed90 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x8600~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x8a00~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ed90 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x8a00~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x8e00~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ed90 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x8e00~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x9200~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9200~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 5#1:10f769a3:::smithi13140796-42:head# 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ed90 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:10f769a3:::smithi13140796-42:head# key 0x85800000000000000110f769a321736d'ithi13140796-42!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ed90 sbid 0x0)) need {<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x9200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.826+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1138 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# = 0 2019-12-07T19:34:04.826+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 0x0~2b000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#1:10f769a3:::smithi13140796-42:head# 0x0~2b000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~2b000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72000 22 blobs 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ed90 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0eda8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84258f30 space 0x558e83d0eda8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate 0x2b000/1000,2b000,0 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x9200~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x48f000~2b000/1000,2b000,0 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x48f000~2b000] 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x9600~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9600~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a2380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a2540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ed90 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a2380 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a2540 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x9600~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e841a2380) get_ref 0x0~2000 Blob(0x558e841a2380 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a2540 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x9a00~200 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e841a2380) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e841a2380 blob([0x48f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2540 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e841a2558 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ed90 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x9a00~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x9e00~200 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423c690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f61f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423c690 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61f80 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8423c690) get_ref 0x0~2000 Blob(0x558e8423c690 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61f80 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8423c690) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ed20 blob([0x13d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ed90 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e8423c690 blob([0x491000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f61f80 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f61f98 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x9e00~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xa200~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa200~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464c000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464c070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464c000 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c070 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c000) get_ref 0x0~2000 Blob(0x558e8464c000 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c070 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c000) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ee70 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e8464c000 blob([0x493000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464c070 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464c088 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ee70 sbid 0x0)) need {<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464c0e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464c150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464c0e0 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c150 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c0e0) get_ref 0x0~2000 Blob(0x558e8464c0e0 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c150 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c0e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e8464c0e0 blob([0x495000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464c150 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464c168 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464c1c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464c230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464c1c0 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c230 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c1c0) get_ref 0x0~2000 Blob(0x558e8464c1c0 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c230 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c1c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e8464c1c0 blob([0x497000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464c230 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464c248 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464c2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464c310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464c2a0 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c310 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c2a0) get_ref 0x0~2000 Blob(0x558e8464c2a0 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c310 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c2a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e8464c2a0 blob([0x499000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464c310 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464c328 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464c380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464c3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464c380 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c3f0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c380) get_ref 0x0~2000 Blob(0x558e8464c380 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c3f0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c380) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e8464c380 blob([0x49b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464c3f0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464c408 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464c460 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464c4d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464c460 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c4d0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c460) get_ref 0x0~2000 Blob(0x558e8464c460 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c4d0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c460) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e8464c460 blob([0x49d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464c4d0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464c4e8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464c540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464c5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464c540 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c5b0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c540) get_ref 0x0~2000 Blob(0x558e8464c540 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c5b0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c540) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e8464c540 blob([0x49f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464c5b0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464c5c8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464c620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464c690 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464c620 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c690 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c620) get_ref 0x0~2000 Blob(0x558e8464c620 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c690 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c620) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e8464c620 blob([0x4a1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464c690 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464c6a8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464c700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464c770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464c700 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c770 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ee70 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c700) get_ref 0x0~2000 Blob(0x558e8464c700 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c770 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c700) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0ee88 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e8464c700 blob([0x4a3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464c770 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84258fc0 space 0x558e83d0ee88 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464c788 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xa200~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xa600~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa600~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464c7e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464c850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ee70 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464c7e0 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c850 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xa600~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c7e0) get_ref 0x0~2000 Blob(0x558e8464c7e0 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c850 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xaa00~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xaa00~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e8464c7e0 blob([0x4a5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464c850 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464c868 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ee70 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xaa00~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464c8c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464c930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xae00~200 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464c8c0 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c930 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xae00~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c8c0) get_ref 0x0~2000 Blob(0x558e8464c8c0 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464c930 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ee70 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e8464c8c0 blob([0x4a7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464c930 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xae00~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464c948 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xb200~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb200~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ee70 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ee70 sbid 0x0)) need {<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464c9a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464ca10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xb200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464c9a0 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464ca10 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c9a0) get_ref 0x0~2000 Blob(0x558e8464c9a0 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464ca10 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464c9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e8464c9a0 blob([0x4a9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464ca10 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464ca28 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464ca80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464caf0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464ca80 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464caf0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464ca80) get_ref 0x0~2000 Blob(0x558e8464ca80 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464caf0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464ca80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e8464ca80 blob([0x4ab000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464caf0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464cb08 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464cb60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464cbd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464cb60 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464cbd0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464cb60) get_ref 0x0~2000 Blob(0x558e8464cb60 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464cbd0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464cb60) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e8464cb60 blob([0x4ad000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464cbd0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464cbe8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464cc40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464ccb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464cc40 blob([0x4af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464ccb0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464cc40) get_ref 0x0~2000 Blob(0x558e8464cc40 blob([0x4af000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464ccb0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464cc40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e8464cc40 blob([0x4af000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464ccb0 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464ccc8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464cd20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464cd90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464cd20 blob([0x4b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464cd90 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464cd20) get_ref 0x0~2000 Blob(0x558e8464cd20 blob([0x4b1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464cd90 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464cd20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e8464cd20 blob([0x4b1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464cd90 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464cda8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464ce00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464ce70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464ce00 blob([0x4b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464ce70 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464ce00) get_ref 0x0~2000 Blob(0x558e8464ce00 blob([0x4b3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464ce70 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464ce00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e8464ce00 blob([0x4b3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464ce70 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464ce88 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464cee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464cf50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464cee0 blob([0x4b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464cf50 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464cee0) get_ref 0x0~2000 Blob(0x558e8464cee0 blob([0x4b5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464cf50 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464cee0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e8464cee0 blob([0x4b5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464cf50 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464cf68 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464cfc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464d030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464cfc0 blob([0x4b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464d030 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464cfc0) get_ref 0x0~2000 Blob(0x558e8464cfc0 blob([0x4b7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464d030 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464cfc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e8464cfc0 blob([0x4b7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464d030 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464d048 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464d0a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464d110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464d0a0 blob([0x4b9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464d110 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464d0a0) get_ref 0x0~1000 Blob(0x558e8464d0a0 blob([0x4b9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464d110 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464d0a0) get_ref init 0x1000, 1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~1000: 0x0~1000 Blob(0x558e8464d0a0 blob([0x4b9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8464d110 sbid 0x0)) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464d128 in 0x558e80219960) _discard 0x0~1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x2b000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e83c73a08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 0x0~2b000 = 0 2019-12-07T19:34:04.826+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 4 keys 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:10f769a3:::smithi13140796-42:head# 4 keys = 0 2019-12-07T19:34:04.826+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ee70 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83d0ee88 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x000000000000045f'.missing/0000000000000001.80FE695C.head.smithi13140796-42..' <- missing/0000000000000001.80FE695C.head.smithi13140796-42.. 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84259050 space 0x558e83d0ee88 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.826+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xb200~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xb600~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb600~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ee70 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xb600~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72000 cost 270483 (23 ios * 4000 + 178483 bytes) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72000 onodes 0x558e83c73900 shared_blobs 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xba00~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xba00~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e83c73a08) update 5#1:10f769a3:::smithi13140796-42:head# 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e83c73a08) update inline shard 418 bytes from 22 extents 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ee70 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xba00~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:10f769a3:::smithi13140796-42:head# is 909 (485 bytes onode + 2 bytes spanning blobs + 422 bytes inline extents) 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xbe00~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xbe00~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72000 allocated 0x[48f000~2b000] released 0x[] 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 freelist allocate 0x48f000~2b000 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 freelist _xor first_key 0x480000 last_key 0x480000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0ee00 blob([0x13f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0ee70 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xbe00~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xc200~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc200~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc53b0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 prepare 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83e72000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc53b0 sbid 0x0)) need {<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc53b0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc53c8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e842590e0 space 0x558e83cc53c8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xc200~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xc600~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc600~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc53b0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xc600~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xca00~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xca00~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc53b0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xca00~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xce00~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xce00~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc53b0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xce00~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xd200~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd200~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc53b0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc53b0 sbid 0x0)) need {<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xd200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc53b0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc53c8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84259170 space 0x558e83cc53c8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xd200~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xd600~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd600~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc53b0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xd600~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xda00~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xda00~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e83b0adc0 finish 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc53b0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xda00~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xde00~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xde00~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0eee0 blob([0x141000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc53b0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.826+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xde00~200 = 512 2019-12-07T19:34:04.826+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xe200~200 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe200~200 size 0x21000 (135168) 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5490 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5490 sbid 0x0)) need {<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.826+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.826+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 23'14 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 21'11 active m=6 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.826+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.827+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 aio_wait 2019-12-07T19:34:04.827+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72000 2019-12-07T19:34:04.827+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 io_done 2019-12-07T19:34:04.827+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.827+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.827+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5490 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc54a8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84259200 space 0x558e83cc54a8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xe200~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xe600~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe600~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5490 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xe600~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xea00~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xea00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5490 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xea00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xee00~200 2019-12-07T19:34:04.827+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2350 < min_flush_size 524288 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xee00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5490 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xee00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xf200~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf200~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5490 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5490 sbid 0x0)) need {<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xf200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.827+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c73900 had 1 2019-12-07T19:34:04.827+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.827+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2373 < min_flush_size 524288 2019-12-07T19:34:04.827+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2373 < min_flush_size 524288 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.827+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.827+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xd10bd~945 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.827+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xd10bd 0xd10bd~945 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.827+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.827+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa10bd 2019-12-07T19:34:04.827+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xbd 2019-12-07T19:34:04.827+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.827+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xa02 and padding block with 0x5fe 2019-12-07T19:34:04.827+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xd1a02 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5490 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.827+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.827+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.827+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc54a8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.827+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84259290 space 0x558e83cc54a8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xf200~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xf600~200 2019-12-07T19:34:04.827+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000138352s (0.000045016s flush + 0.000093336s kv commit) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf600~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e72000> 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.827+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 kv_submitted 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5490 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.827+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 finishing 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72000 onodes 0x558e83c73900 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xf600~200 = 512 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f61f98 in 0x558e80219960) _finish_write discard buffer(0x558e84259710 space 0x558e83f61f98 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xfa00~200 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a2558 in 0x558e80219960) _finish_write discard buffer(0x558e84259680 space 0x558e841a2558 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464c088 in 0x558e80219960) _finish_write discard buffer(0x558e842597a0 space 0x558e8464c088 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfa00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464c168 in 0x558e80219960) _finish_write discard buffer(0x558e84259830 space 0x558e8464c168 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464c248 in 0x558e80219960) _finish_write discard buffer(0x558e842598c0 space 0x558e8464c248 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464c328 in 0x558e80219960) _finish_write discard buffer(0x558e84259950 space 0x558e8464c328 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5490 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xfa00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464c408 in 0x558e80219960) _finish_write discard buffer(0x558e842599e0 space 0x558e8464c408 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xfe00~200 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464c4e8 in 0x558e80219960) _finish_write discard buffer(0x558e84259a70 space 0x558e8464c4e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464c5c8 in 0x558e80219960) _finish_write discard buffer(0x558e84259b00 space 0x558e8464c5c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464c6a8 in 0x558e80219960) _finish_write discard buffer(0x558e84259b90 space 0x558e8464c6a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464c788 in 0x558e80219960) _finish_write discard buffer(0x558e84259c20 space 0x558e8464c788 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfe00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464c868 in 0x558e80219960) _finish_write discard buffer(0x558e84259cb0 space 0x558e8464c868 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464c948 in 0x558e80219960) _finish_write discard buffer(0x558e84259d40 space 0x558e8464c948 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5420 blob([0x143000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5490 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0xfe00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464ca28 in 0x558e80219960) _finish_write discard buffer(0x558e84259dd0 space 0x558e8464ca28 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464cb08 in 0x558e80219960) _finish_write discard buffer(0x558e84259e60 space 0x558e8464cb08 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x10200~200 2019-12-07T19:34:04.827+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 23'14 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 21'11 active m=6 mbc={} ps=[2~1]] recovery_committed_to version 23'14 now ondisk 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10200~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464cbe8 in 0x558e80219960) _finish_write discard buffer(0x558e84259ef0 space 0x558e8464cbe8 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464ccc8 in 0x558e80219960) _finish_write discard buffer(0x558e84670000 space 0x558e8464ccc8 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464cda8 in 0x558e80219960) _finish_write discard buffer(0x558e84670090 space 0x558e8464cda8 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5570 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464ce88 in 0x558e80219960) _finish_write discard buffer(0x558e84670120 space 0x558e8464ce88 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464cf68 in 0x558e80219960) _finish_write discard buffer(0x558e846701b0 space 0x558e8464cf68 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464d048 in 0x558e80219960) _finish_write discard buffer(0x558e84670240 space 0x558e8464d048 0x0~2000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5570 sbid 0x0)) need {<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464d128 in 0x558e80219960) _finish_write discard buffer(0x558e846702d0 space 0x558e8464d128 0x0~1000 writing nocache) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.827+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 27/26 [PushReplyOp(1:10f769a3:::smithi13140796-42:head)]) v3 -- 0x558e84197000 con 0x558e818d7c00 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72000 done 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.827+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72000 [] 2019-12-07T19:34:04.827+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.827+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.827+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5570 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5588 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84259320 space 0x558e83cc5588 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x10200~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x10600~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10600~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5570 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x10600~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x10a00~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5570 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x10a00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x10e00~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5570 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x10e00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x11200~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11200~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5570 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5570 sbid 0x0)) need {<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x11200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5570 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5588 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e842593b0 space 0x558e83cc5588 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x11200~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x11600~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11600~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5570 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x11600~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x11a00~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5570 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x11a00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x11e00~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5500 blob([0x145000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5570 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x11e00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x12200~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12200~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5650 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5650 sbid 0x0)) need {<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x12200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5650 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5668 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84259440 space 0x558e83cc5668 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x12200~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x12600~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12600~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5650 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x12600~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x12a00~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5650 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x12a00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x12e00~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5650 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x12e00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x13200~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13200~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5650 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5650 sbid 0x0)) need {<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x13200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5650 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5668 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e842594d0 space 0x558e83cc5668 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x13200~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x13600~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13600~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5650 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x13600~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x13a00~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5650 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x13a00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x13e00~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc55e0 blob([0x147000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5650 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x13e00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x14200~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14200~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5730 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5730 sbid 0x0)) need {<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x14200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5730 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5748 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84259560 space 0x558e83cc5748 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x14200~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x14600~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14600~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5730 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x14600~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x14a00~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5730 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x14a00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x14e00~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5730 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x14e00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x15200~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15200~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5730 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5730 sbid 0x0)) need {<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x15200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5730 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5748 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e842595f0 space 0x558e83cc5748 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x15200~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x15600~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15600~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5730 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x15600~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x15a00~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5730 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x15a00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x15e00~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc56c0 blob([0x149000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5730 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.827+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x15e00~200 = 512 2019-12-07T19:34:04.827+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x16200~200 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16200~200 size 0x21000 (135168) 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5810 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5810 sbid 0x0)) need {<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x16200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.827+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5810 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5828 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84670000 space 0x558e83cc5828 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x16200~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x16600~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16600~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5810 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x16600~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x16a00~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5810 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x16a00~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x16e00~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5810 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x16e00~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x17200~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17200~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5810 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5810 sbid 0x0)) need {<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x17200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5810 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5828 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84670090 space 0x558e83cc5828 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x17200~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x17600~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17600~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5810 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x17600~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x17a00~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5810 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x17a00~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x17e00~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc57a0 blob([0x14b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5810 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x17e00~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x18200~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18200~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc58f0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc58f0 sbid 0x0)) need {<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x18200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc58f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5908 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84670120 space 0x558e83cc5908 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x18200~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x18600~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18600~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc58f0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x18600~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x18a00~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc58f0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x18a00~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x18e00~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc58f0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x18e00~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x19200~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19200~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc58f0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc58f0 sbid 0x0)) need {<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x19200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc58f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5908 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e846701b0 space 0x558e83cc5908 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x19200~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x19600~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19600~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc58f0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x19600~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x19a00~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc58f0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x19a00~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x19e00~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5880 blob([0x14d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc58f0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x19e00~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1a200~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a200~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc59d0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc59d0 sbid 0x0)) need {<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1a200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc59d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc59e8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84670240 space 0x558e83cc59e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1a200~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1a600~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a600~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc59d0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1a600~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1aa00~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1aa00~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc59d0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1aa00~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1ae00~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ae00~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc59d0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1ae00~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1b200~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b200~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc59d0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc59d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1b200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc59d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc59e8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84670870 space 0x558e83cc59e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1b200~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1b600~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b600~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc59d0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1b600~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1ba00~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ba00~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc59d0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1ba00~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1be00~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1be00~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5960 blob([0x14f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc59d0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1be00~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1c200~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c200~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5ab0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5ab0 sbid 0x0)) need {<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1c200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5ab0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5ac8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84670900 space 0x558e83cc5ac8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1c200~200 = 512 2019-12-07T19:34:04.828+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1c600~200 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c600~200 size 0x21000 (135168) 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.828+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5ab0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.828+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1c600~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1ca00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ca00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5ab0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1ca00~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1ce00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ce00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5ab0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1ce00~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1d200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5ab0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5ab0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1d200:1200~200]} 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1d200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5ab0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1d200:1200~200]} 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5ac8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84670990 space 0x558e83cc5ac8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1d200~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1d600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5ab0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1d600~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1da00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1da00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5ab0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1da00~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1de00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1de00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5a40 blob([0x151000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83cc5ab0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1de00~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1e200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e770 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e770 sbid 0x0)) need {<0x0, 0x1000> : [0x1e200:200~200]} 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1e200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e770 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1e200:200~200]} 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e8194e788 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84670a20 space 0x558e8194e788 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1e200~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1e600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e770 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1e600~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1ea00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ea00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e770 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1ea00~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1ee00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ee00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e770 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1ee00~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1f200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e770 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e770 sbid 0x0)) need {<0x1000, 0x1000> : [0x1f200:1200~200]} 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1f200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e770 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1f200:1200~200]} 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e8194e788 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84670ab0 space 0x558e8194e788 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1f200~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1f600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e770 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1f600~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1fa00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fa00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e770 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1fa00~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1fe00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fe00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5b20 blob([0x153000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e770 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x1fe00~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x20200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5b90 blob([0x155000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83cc5c00 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83cc5b90 blob([0x155000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83cc5c00 sbid 0x0)) need {<0x0, 0x1000> : [0x20200:200~200]} 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x20200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83cc5b90 blob([0x155000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83cc5c00 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x20200:200~200]} 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83cc5c18 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84670b40 space 0x558e83cc5c18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x20200~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x20600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5b90 blob([0x155000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83cc5c00 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x20600~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x20a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5b90 blob([0x155000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83cc5c00 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x20a00~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x20e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83cc5b90 blob([0x155000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83cc5c00 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x20e00~200 = 512 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x21200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x21200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x21600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x21600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x21a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x21a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x21e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x21e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x22200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x22200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x22600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x22600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x22a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x22a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x22e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x22e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x23200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x23200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x23600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x23600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x23a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x23a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x23e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x23e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x24200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x24200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x24600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x24600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x24a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x24a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x24e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x24e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x25200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x25200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x25600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x25600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x25a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x25a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x25e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x25e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x26200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x26200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x26600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x26600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x26a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x26a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x26e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x26e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x27200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x27200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x27600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x27600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x27a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x27a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x27e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x27e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x28200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x28200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x28600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x28600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x28a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x28a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x28e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x28e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x29200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x29200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x29600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x29600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x29a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x29a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x29e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x29e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2a200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2a200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2a600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2a600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2aa00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2aa00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2aa00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2ae00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ae00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2ae00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2b200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2b200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2b600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2b600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2ba00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ba00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2ba00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2be00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2be00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2be00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2c200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2c200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2c600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2c600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2ca00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ca00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2ca00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2ce00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ce00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2ce00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2d200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2d200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2d600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2d600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2da00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2da00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2da00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2de00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2de00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2de00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2e200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2e200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2e600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2e600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2ea00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ea00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2ea00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2ee00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ee00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2ee00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2f200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2f200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2f600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2f600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2fa00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fa00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2fa00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2fe00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fe00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x2fe00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x30200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x30200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x30600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x30600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x30a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x30a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x30e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x30e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x31200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x31200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x31600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x31600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x31a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x31a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x31e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x31e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x32200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x32200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x32600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x32600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x32a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x32a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x32e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x32e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x33200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x33200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x33600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x33600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x33a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x33a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x33e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x33e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x34200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x34200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x34600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x34600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x34a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x34a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x34e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x34e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x35200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x35200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x35600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x35600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x35a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x35a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x35e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x35e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x36200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x36200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x36600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x36600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x36a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x36a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x36e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x36e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x37200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x37200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x37600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x37600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x37a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x37a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x37e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x37e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x38200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x38200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x38600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x38600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x38a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x38a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x38e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x38e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x39200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x39200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x39600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x39600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x39a00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39a00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x39a00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x39e00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39e00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x39e00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3a200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3a200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3a600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3a600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3aa00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3aa00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3aa00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3ae00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ae00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3ae00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3b200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3b200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3b600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3b600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3ba00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ba00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3ba00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3be00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3be00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3be00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3c200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3c200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3c600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3c600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3ca00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ca00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3ca00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3ce00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ce00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3ce00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3d200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3d200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3d600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3d600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3da00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3da00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3da00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3de00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3de00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3de00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3e200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3e200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3e600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3e600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3ea00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ea00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3ea00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3ee00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ee00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3ee00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3f200~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f200~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3f200~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3f600~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f600~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3f600~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3fa00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fa00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3fa00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3fe00~200 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fe00~200 size 0x21000 (135168) 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 0x3fe00~200 = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 20 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=67584 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:fc3a86a2:::smithi13140796-33:head 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:fc3a86a2:::smithi13140796-33:head 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 135168 2019-12-07T19:34:04.829+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# hinfo_key 2019-12-07T19:34:04.829+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:fc3a86a2:::smithi13140796-33:head# hinfo_key = 0 2019-12-07T19:34:04.829+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.fs0 27/26 ECSubReadReply(tid=80, attrs_read=0)) v2 -- 0x558e8402e500 con 0x558e818d7000 2019-12-07T19:34:04.830+0000 7fe2c8267700 10 osd.0 27 dequeue_op 0x558e83b0b340 finish 2019-12-07T19:34:04.830+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.836+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 171 ==== MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:4, version: 27'15, data_included: [0~131072], data_size: 131072, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:4@27'15, size: 524288, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:524288, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 132110+0+0 (unknown 0 0 0) 0x558e81965200 con 0x558e818d7c00 2019-12-07T19:34:04.836+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e841774a0 prio 127 cost 132072 latency 0.000042 epoch 27 MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:4, version: 27'15, data_included: [0~131072], data_size: 131072, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:4@27'15, size: 524288, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:524288, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.836+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:4, version: 27'15, data_included: [0~131072], data_size: 131072, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:4@27'15, size: 524288, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:524288, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 132072 e27) 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:4, version: 27'15, data_included: [0~131072], data_size: 131072, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:4@27'15, size: 524288, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:524288, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 132072 e27) queued 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:4, version: 27'15, data_included: [0~131072], data_size: 131072, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:4@27'15, size: 524288, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:524288, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 132072 e27) pg 0x558e841a4000 2019-12-07T19:34:04.836+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 97 ==== MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=81, to_read={1:f7919c75:::smithi13140796-20:4=0,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (unknown 0 0 0) 0x558e8402e500 con 0x558e818d7000 2019-12-07T19:34:04.836+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e81943080 prio 3 cost 0 latency 0.000027 epoch 27 MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=81, to_read={1:f7919c75:::smithi13140796-20:4=0,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.836+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e841774a0 prio 127 cost 132072 latency 0.000111 MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:4, version: 27'15, data_included: [0~131072], data_size: 131072, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:4@27'15, size: 524288, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:524288, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 27'23 lc 23'14 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=6 mbc={} ps=[2~1]] 2019-12-07T19:34:04.836+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=81, to_read={1:f7919c75:::smithi13140796-20:4=0,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.836+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 23'14 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=6 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:4, version: 27'15, data_included: [0~131072], data_size: 131072, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:4@27'15, size: 524288, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:524288, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=81, to_read={1:f7919c75:::smithi13140796-20:4=0,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.836+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 23'14 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=6 mbc={} ps=[2~1]] on_local_recover: 1:10779340:::smithi13140796-32:4 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 snap_mapper.remove_oid 1:10779340:::smithi13140796-32:4 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 snap_mapper._remove_oid 1:10779340:::smithi13140796-32:4 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=81, to_read={1:f7919c75:::smithi13140796-20:4=0,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.836+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:04.836+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e81943080 prio 3 cost 0 latency 0.000076 MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=81, to_read={1:f7919c75:::smithi13140796-20:4=0,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.836+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 snap_mapper.get_snaps 1:10779340:::smithi13140796-32:4 got.empty() 2019-12-07T19:34:04.836+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=81, to_read={1:f7919c75:::smithi13140796-20:4=0,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:4=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 23'14 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=6 mbc={} ps=[2~1]] snapset 4=[]:{4=[4]} 2019-12-07T19:34:04.836+0000 7fe2c425f700 25 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 23'14 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=6 mbc={} ps=[2~1]] snaps 4 2019-12-07T19:34:04.836+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x200~200 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 snap_mapper.add_oid 1:10779340:::smithi13140796-32:4 4 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x200~200 size 0x14000 (81920) 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.836+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:04.836+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95650 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 snap_mapper.get_snaps 1:10779340:::smithi13140796-32:4 got.empty() 2019-12-07T19:34:04.836+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x200~200 = 512 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 snap_mapper.set_snaps 1:10779340:::smithi13140796-32:4 4 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.80EE9C20.4.smithi13140796-32.. 2019-12-07T19:34:04.836+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x600~200 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x600~200 size 0x14000 (81920) 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95650 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 snap_mapper.add_oid set SNA_1_0000000000000004_.5_0000000000000001.80EE9C20.4.smithi13140796-32.. 2019-12-07T19:34:04.836+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x600~200 = 512 2019-12-07T19:34:04.836+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xa00~200 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa00~200 size 0x14000 (81920) 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95650 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.836+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xa00~200 = 512 2019-12-07T19:34:04.836+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xe00~200 2019-12-07T19:34:04.836+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 23'14 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=6 mbc={} ps=[2~1]] got missing 1:10779340:::smithi13140796-32:4 v 27'15 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe00~200 size 0x14000 (81920) 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95650 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.836+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xe00~200 = 512 2019-12-07T19:34:04.836+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1200~200 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1200~200 size 0x14000 (81920) 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.836+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'17 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 23'14 active m=5 mbc={} ps=[2~1]] last_complete now 27'17 log.complete_to 27'18 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95650 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95650 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need {<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.836+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.836+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e83e72300 seq 16 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:10779340:::smithi13140796-32:4# key 0x858000000000000001107793'@!smithi13140796-32!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.836+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 5#1:10779340:::smithi13140796-32:4# 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:10779340:::smithi13140796-32:4# key 0x858000000000000001107793'@!smithi13140796-32!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.836+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:10779340:::smithi13140796-32:4# 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1139 2019-12-07T19:34:04.836+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:10779340:::smithi13140796-32:4# = 0 2019-12-07T19:34:04.836+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:10779340:::smithi13140796-32:4# 0x0~20000 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#1:10779340:::smithi13140796-32:4# 0x0~20000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.836+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~20000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72300 16 blobs 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95650 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0x{<0x1000, 0x1000> : [0x1200:1200~200]} 2019-12-07T19:34:04.836+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate 0x20000/1000,20000,0 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e8194e4e8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.836+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x4ba000~20000/1000,20000,0 2019-12-07T19:34:04.836+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84258750 space 0x558e8194e4e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x4ba000~20000] 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f61030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f61f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1200~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1600~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1600~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f61030 blob([0x4ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61f10 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95650 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f61030) get_ref 0x0~2000 Blob(0x558e83f61030 blob([0x4ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61f10 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f61030) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1600~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1a00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95650 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1a00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1e00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95650 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1e00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2200~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2200~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83f61030 blob([0x4ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f61f10 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f61f28 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need {<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed4070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464db90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed4070 blob([0x4bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464db90 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e83ed4070) get_ref 0x0~2000 Blob(0x558e83ed4070 blob([0x4bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464db90 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e83ed4070) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83ed4070 blob([0x4bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464db90 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464dba8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464dc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464dc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464dc00 blob([0x4be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464dc70 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464dc00) get_ref 0x0~2000 Blob(0x558e8464dc00 blob([0x4be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464dc70 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464dc00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e8464dc00 blob([0x4be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464dc70 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464dc88 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464dce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464dd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464dce0 blob([0x4c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464dd50 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464dce0) get_ref 0x0~2000 Blob(0x558e8464dce0 blob([0x4c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464dd50 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464dce0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e8464dce0 blob([0x4c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464dd50 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464dd68 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464ddc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464de30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464ddc0 blob([0x4c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464de30 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464ddc0) get_ref 0x0~2000 Blob(0x558e8464ddc0 blob([0x4c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464de30 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464ddc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e8464ddc0 blob([0x4c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464de30 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464de48 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464dea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464df10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464dea0 blob([0x4c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464df10 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464dea0) get_ref 0x0~2000 Blob(0x558e8464dea0 blob([0x4c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464df10 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464dea0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e8464dea0 blob([0x4c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464df10 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464df28 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464df80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456a000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464df80 blob([0x4c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a000 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464df80) get_ref 0x0~2000 Blob(0x558e8464df80 blob([0x4c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a000 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464df80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e8464df80 blob([0x4c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a000 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a018 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0x{<0x0, 0x1000> : [0x2200:200~200]} 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456a0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a070 blob([0x4c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a0e0 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e8194f828 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842587e0 space 0x558e8194f828 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a070) get_ref 0x0~2000 Blob(0x558e8456a070 blob([0x4c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a0e0 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a070) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2200~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e8456a070 blob([0x4c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a0e0 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a0f8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2600~200 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456a1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2600~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a150 blob([0x4ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a1c0 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a150) get_ref 0x0~2000 Blob(0x558e8456a150 blob([0x4ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a1c0 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a150) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2600~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e8456a150 blob([0x4ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a1c0 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2a00~200 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a1d8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456a2a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a230 blob([0x4cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a2a0 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2a00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a230) get_ref 0x0~2000 Blob(0x558e8456a230 blob([0x4cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a2a0 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a230) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2e00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e8456a230 blob([0x4cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a2a0 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a2b8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456a380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2e00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a310 blob([0x4ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a380 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3200~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3200~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a310) get_ref 0x0~2000 Blob(0x558e8456a310 blob([0x4ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a380 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a310) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e8456a310 blob([0x4ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a380 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a398 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need {<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456a460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a3f0 blob([0x4d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a460 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a3f0) get_ref 0x0~2000 Blob(0x558e8456a3f0 blob([0x4d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a460 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e8456a3f0 blob([0x4d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a460 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a478 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456a540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a4d0 blob([0x4d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a540 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a4d0) get_ref 0x0~2000 Blob(0x558e8456a4d0 blob([0x4d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a540 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e8456a4d0 blob([0x4d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a540 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a558 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a5b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456a620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a5b0 blob([0x4d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a620 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a5b0) get_ref 0x0~2000 Blob(0x558e8456a5b0 blob([0x4d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a620 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a5b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e8456a5b0 blob([0x4d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a620 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a638 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456a700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a690 blob([0x4d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a700 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a690) get_ref 0x0~2000 Blob(0x558e8456a690 blob([0x4d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a700 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a690) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e8456a690 blob([0x4d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a700 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a718 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456a7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a770 blob([0x4d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a7e0 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a770) get_ref 0x0~2000 Blob(0x558e8456a770 blob([0x4d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a7e0 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a770) get_ref init 0x2000, 1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e8456a770 blob([0x4d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a7e0 sbid 0x0)) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a7f8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x20000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e8417d508) dirty_range mark inline shard dirty 2019-12-07T19:34:04.837+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:10779340:::smithi13140796-32:4# 0x0~20000 = 0 2019-12-07T19:34:04.837+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:4# 3 keys 2019-12-07T19:34:04.837+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:4# 3 keys = 0 2019-12-07T19:34:04.837+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.80EE9C20.4.smithi13140796-32..' <- OBJ_.5_0000000000000001.80EE9C20.4.smithi13140796-32.. 2019-12-07T19:34:04.837+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.837+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.5_0000000000000001.80EE9C20.4.smithi13140796-32..' <- SNA_1_0000000000000004_.5_0000000000000001.80EE9C20.4.smithi13140796-32.. 2019-12-07T19:34:04.837+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.837+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x000000000000045f'.missing/0000000000000001.80EE9C20.4.smithi13140796-32..' <- missing/0000000000000001.80EE9C20.4.smithi13140796-32.. 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0x{<0x1000, 0x1000> : [0x3200:1200~200]} 2019-12-07T19:34:04.837+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e8194f828 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84258870 space 0x558e8194f828 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3200~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3600~200 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3600~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72300 cost 201891 (17 ios * 4000 + 133891 bytes) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72300 onodes 0x558e8417d400 shared_blobs 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e8417d508) update 5#1:10779340:::smithi13140796-32:4# 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3600~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3a00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e8417d508) update inline shard 307 bytes from 16 extents 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3a00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:10779340:::smithi13140796-32:4# is 725 (412 bytes onode + 2 bytes spanning blobs + 311 bytes inline extents) 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3e00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72300 allocated 0x[4ba000~20000] released 0x[] 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.837+0000 7fe2c7a66700 10 freelist allocate 0x4ba000~20000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 freelist _xor first_key 0x480000 last_key 0x480000 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3e00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x4200~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4200~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need {<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.837+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72300 prepare 2019-12-07T19:34:04.837+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83e72300 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0x{<0x0, 0x1000> : [0x4200:200~200]} 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e8194e8d8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84671290 space 0x558e8194e8d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x4200~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x4600~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4600~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x4600~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x4a00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x4a00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x4e00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x4e00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x5200~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5200~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need {<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x5200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0x{<0x1000, 0x1000> : [0x5200:1200~200]} 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e8194e8d8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84671320 space 0x558e8194e8d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x5200~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x5600~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5600~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x5600~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x5a00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x5a00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x5e00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x5e00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x6200~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6200~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need {<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0x{<0x0, 0x1000> : [0x6200:200~200]} 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83c65b38 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e846713b0 space 0x558e83c65b38 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x6200~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x6600~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6600~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e841774a0 finish 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x6600~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x6a00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x6a00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x6e00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'17 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 23'14 active m=5 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x6e00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x7200~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7200~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need {<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x7200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0x{<0x1000, 0x1000> : [0x7200:1200~200]} 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83c65b38 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84671440 space 0x558e83c65b38 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x7200~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x7600~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7600~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x7600~200 = 512 2019-12-07T19:34:04.837+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72300 aio_wait 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x7a00~200 2019-12-07T19:34:04.837+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72300 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.837+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72300 io_done 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x7a00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x7e00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x7e00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x8200~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8200~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need {<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0x{<0x0, 0x1000> : [0x8200:200~200]} 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83c65ac8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e846714d0 space 0x558e83c65ac8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x8200~200 = 512 2019-12-07T19:34:04.837+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2478 < min_flush_size 524288 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x8600~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8600~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x8600~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x8a00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x8a00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x8e00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x8e00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x9200~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9200~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need {<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417d400 had 1 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x9200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.837+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2501 < min_flush_size 524288 2019-12-07T19:34:04.837+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2501 < min_flush_size 524288 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.837+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.837+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xd1a02~9c5 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.837+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xd1a02 0xd1a02~9c5 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa1a02 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xa02 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x3c7 and padding block with 0xc39 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0x{<0x1000, 0x1000> : [0x9200:1200~200]} 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83c65ac8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84671560 space 0x558e83c65ac8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x9200~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x9600~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9600~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x9600~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x9a00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x9a00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x9e00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x9e00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xa200~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa200~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need {<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xd23c7 2019-12-07T19:34:04.837+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.837+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.837+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000164186s (0.000041541s flush + 0.000122645s kv commit) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.837+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e72300> 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.837+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72300 kv_submitted 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72300 2019-12-07T19:34:04.837+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72300 finishing 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72300 onodes 0x558e8417d400 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f61f28 in 0x558e80219960) _finish_write discard buffer(0x558e84032d80 space 0x558e83f61f28 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456a018 in 0x558e80219960) _finish_write discard buffer(0x558e84670750 space 0x558e8456a018 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456a0f8 in 0x558e80219960) _finish_write discard buffer(0x558e846707e0 space 0x558e8456a0f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456a1d8 in 0x558e80219960) _finish_write discard buffer(0x558e84670e10 space 0x558e8456a1d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456a2b8 in 0x558e80219960) _finish_write discard buffer(0x558e84670ea0 space 0x558e8456a2b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456a398 in 0x558e80219960) _finish_write discard buffer(0x558e84670f30 space 0x558e8456a398 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456a478 in 0x558e80219960) _finish_write discard buffer(0x558e84670fc0 space 0x558e8456a478 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456a558 in 0x558e80219960) _finish_write discard buffer(0x558e84671050 space 0x558e8456a558 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456a638 in 0x558e80219960) _finish_write discard buffer(0x558e846710e0 space 0x558e8456a638 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456a718 in 0x558e80219960) _finish_write discard buffer(0x558e84671170 space 0x558e8456a718 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456a7f8 in 0x558e80219960) _finish_write discard buffer(0x558e84671200 space 0x558e8456a7f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'17 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 23'14 active m=5 mbc={} ps=[2~1]] recovery_committed_to version 27'17 now ondisk 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464dba8 in 0x558e80219960) _finish_write discard buffer(0x558e84670480 space 0x558e8464dba8 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464dc88 in 0x558e80219960) _finish_write discard buffer(0x558e84670510 space 0x558e8464dc88 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464dd68 in 0x558e80219960) _finish_write discard buffer(0x558e846705a0 space 0x558e8464dd68 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464de48 in 0x558e80219960) _finish_write discard buffer(0x558e84670630 space 0x558e8464de48 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464df28 in 0x558e80219960) _finish_write discard buffer(0x558e846706c0 space 0x558e8464df28 0x0~2000 writing nocache) 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72300 done 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.837+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72300 [] 2019-12-07T19:34:04.837+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.837+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 27/26 [PushReplyOp(1:10779340:::smithi13140796-32:4)]) v3 -- 0x558e818ee600 con 0x558e818d7c00 2019-12-07T19:34:04.837+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0x{<0x0, 0x1000> : [0xa200:200~200]} 2019-12-07T19:34:04.837+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83c65c18 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e846715f0 space 0x558e83c65c18 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xa200~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xa600~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa600~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xa600~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xaa00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xaa00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xaa00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xae00~200 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xae00~200 size 0x14000 (81920) 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.837+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.837+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xae00~200 = 512 2019-12-07T19:34:04.837+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xb200~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb200~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need {<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xb200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0x{<0x1000, 0x1000> : [0xb200:1200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83c65c18 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84671680 space 0x558e83c65c18 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xb200~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xb600~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb600~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xb600~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xba00~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xba00~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xba00~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xbe00~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xbe00~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xbe00~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xc200~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc200~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need {<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0x{<0x0, 0x1000> : [0xc200:200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af92e8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84671710 space 0x558e83af92e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xc200~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xc600~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc600~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xc600~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xca00~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xca00~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xca00~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xce00~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xce00~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xce00~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xd200~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd200~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need {<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xd200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0x{<0x1000, 0x1000> : [0xd200:1200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af92e8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e846717a0 space 0x558e83af92e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xd200~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xd600~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd600~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xd600~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xda00~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xda00~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xda00~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xde00~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xde00~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xde00~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xe200~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe200~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need {<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0x{<0x0, 0x1000> : [0xe200:200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af93c8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84671830 space 0x558e83af93c8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xe200~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xe600~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe600~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xe600~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xea00~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xea00~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xea00~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xee00~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xee00~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xee00~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xf200~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf200~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need {<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xf200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0x{<0x1000, 0x1000> : [0xf200:1200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af93c8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e846718c0 space 0x558e83af93c8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xf200~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xf600~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf600~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xf600~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xfa00~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfa00~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xfa00~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xfe00~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfe00~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0xfe00~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x10200~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10200~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need {<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0x{<0x0, 0x1000> : [0x10200:200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af94a8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84671950 space 0x558e83af94a8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x10200~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x10600~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10600~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x10600~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x10a00~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x10a00~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x10e00~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x10e00~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x11200~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11200~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need {<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x11200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0x{<0x1000, 0x1000> : [0x11200:1200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af94a8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e846719e0 space 0x558e83af94a8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x11200~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x11600~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11600~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x11600~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x11a00~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x11a00~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x11e00~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.838+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x11e00~200 = 512 2019-12-07T19:34:04.838+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x12200~200 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12200~200 size 0x14000 (81920) 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need {<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x12200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.838+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0x{<0x0, 0x1000> : [0x12200:200~200]} 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af9588 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84671a70 space 0x558e83af9588 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x12200~200 = 512 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x12600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x12600~200 = 512 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x12a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x12a00~200 = 512 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x12e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x12e00~200 = 512 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x13200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need {<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x13200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0x{<0x1000, 0x1000> : [0x13200:1200~200]} 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af9588 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84671b00 space 0x558e83af9588 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x13200~200 = 512 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x13600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x13600~200 = 512 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x13a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x13a00~200 = 512 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x13e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x13e00~200 = 512 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x14200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x14200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x14600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x14600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x14a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x14a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x14e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x14e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x15200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x15200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x15600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x15600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x15a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x15a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x15e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x15e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x16200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x16200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x16600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x16600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x16a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x16a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x16e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x16e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x17200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x17200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x17600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x17600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x17a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x17a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x17e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x17e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x18200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x18200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x18600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x18600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x18a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x18a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x18e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x18e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x19200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x19200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x19600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x19600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x19a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x19a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x19e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x19e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1a200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1a200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1a600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1a600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1aa00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1aa00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1aa00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1ae00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ae00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1ae00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1b200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1b200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1b600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1b600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1ba00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ba00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1ba00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1be00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1be00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1be00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1c200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1c200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1c600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1c600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1ca00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ca00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1ca00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1ce00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ce00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1ce00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1d200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1d200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1d600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1d600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1da00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1da00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1da00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1de00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1de00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1de00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1e200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1e200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1e600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1e600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1ea00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ea00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1ea00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1ee00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ee00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1ee00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1f200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1f200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1f600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1f600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1fa00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fa00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1fa00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1fe00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fe00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x1fe00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x20200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x20200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x20600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x20600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x20a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x20a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x20e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x20e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x21200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x21200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x21600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x21600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x21a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x21a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x21e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x21e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x22200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x22200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x22600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x22600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x22a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x22a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x22e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x22e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x23200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x23200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x23600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x23600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x23a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x23a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x23e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x23e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x24200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x24200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x24600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x24600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x24a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x24a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x24e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x24e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x25200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x25200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x25600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x25600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x25a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x25a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x25e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x25e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x26200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x26200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x26600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x26600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x26a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x26a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x26e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x26e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x27200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x27200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x27600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x27600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x27a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x27a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x27e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x27e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x28200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x28200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x28600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x28600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x28a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x28a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x28e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x28e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x29200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x29200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x29600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x29600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x29a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x29a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x29e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x29e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2a200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2a200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2a600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2a600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2aa00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2aa00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2aa00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2ae00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ae00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2ae00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2b200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2b200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2b600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2b600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2ba00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ba00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2ba00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2be00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2be00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2be00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2c200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2c200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2c600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2c600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2ca00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ca00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2ca00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2ce00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ce00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2ce00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2d200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2d200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2d600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2d600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2da00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2da00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2da00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2de00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2de00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2de00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2e200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2e200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2e600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2e600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2ea00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ea00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2ea00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2ee00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ee00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2ee00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2f200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2f200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2f600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2f600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2fa00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fa00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2fa00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2fe00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fe00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x2fe00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x30200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x30200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x30600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x30600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x30a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x30a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x30e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x30e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x31200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x31200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x31600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x31600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x31a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x31a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x31e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x31e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x32200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x32200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x32600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x32600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x32a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x32a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x32e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x32e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x33200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x33200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x33600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x33600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x33a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x33a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x33e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x33e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x34200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x34200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x34600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x34600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x34a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x34a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x34e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x34e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x35200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x35200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x35600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x35600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x35a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x35a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x35e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x35e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x36200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x36200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x36600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x36600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x36a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x36a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x36e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x36e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x37200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x37200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x37600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x37600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x37a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x37a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x37e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x37e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x38200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x38200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x38600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x38600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x38a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x38a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x38e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x38e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x39200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x39200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x39600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x39600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x39a00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39a00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x39a00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x39e00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39e00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x39e00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3a200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3a200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3a600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3a600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3aa00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3aa00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3aa00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3ae00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ae00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3ae00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3b200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3b200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3b600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3b600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3ba00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ba00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3ba00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3be00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3be00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3be00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3c200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3c200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3c600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3c600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3ca00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ca00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3ca00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3ce00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ce00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3ce00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3d200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3d200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3d600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3d600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3da00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3da00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3da00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3de00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3de00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3de00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3e200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3e200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3e600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3e600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3ea00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ea00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3ea00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3ee00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ee00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3ee00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3f200~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f200~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3f200~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3f600~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f600~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3f600~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3fa00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fa00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3fa00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3fe00~200 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fe00~200 size 0x14000 (81920) 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 0x3fe00~200 = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=40960 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:f7919c75:::smithi13140796-20:4 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:f7919c75:::smithi13140796-20:4 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 81920 2019-12-07T19:34:04.839+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# hinfo_key 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# hinfo_key = 0 2019-12-07T19:34:04.839+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.fs0 27/26 ECSubReadReply(tid=81, attrs_read=0)) v2 -- 0x558e8417d180 con 0x558e818d7000 2019-12-07T19:34:04.839+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e81943080 finish 2019-12-07T19:34:04.839+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.847+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 98 ==== MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=82, to_read={1:f7919c75:::smithi13140796-20:head=0,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (unknown 0 0 0) 0x558e8417d180 con 0x558e818d7000 2019-12-07T19:34:04.847+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e81943600 prio 3 cost 0 latency 0.000029 epoch 27 MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=82, to_read={1:f7919c75:::smithi13140796-20:head=0,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.847+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=82, to_read={1:f7919c75:::smithi13140796-20:head=0,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=82, to_read={1:f7919c75:::smithi13140796-20:head=0,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=82, to_read={1:f7919c75:::smithi13140796-20:head=0,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 osd.0 27 dequeue_op 0x558e81943600 prio 3 cost 0 latency 0.000091 MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=82, to_read={1:f7919c75:::smithi13140796-20:head=0,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=82, to_read={1:f7919c75:::smithi13140796-20:head=0,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.847+0000 7fe2c8267700 25 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac49a0 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac49a0 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xa00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac49a0 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xa00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xe00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac49a0 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xe00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac49a0 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac49a0 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1a00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac49a0 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1a00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1e00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac49a0 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1e00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866af0 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866af0 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2a00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866af0 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2a00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2e00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866af0 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2e00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866af0 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866af0 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3a00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866af0 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3a00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3e00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866af0 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3e00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f570 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f570 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4a00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f570 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4a00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4e00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f570 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4e00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f570 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f570 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5a00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f570 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5a00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5e00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f570 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5e00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65b90 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65b90 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6a00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65b90 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6a00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6e00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65b90 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6e00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65b90 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65b90 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7a00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65b90 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7a00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7e00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65b90 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7e00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x8200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65a40 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x8200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x8600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65a40 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x8600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x8a00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65a40 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x8a00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x8e00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x8e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65a40 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x8e00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x9200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65a40 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x9200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x9600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65a40 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x9600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x9a00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65a40 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x9a00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x9e00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x9e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65a40 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x9e00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xa200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64d20 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xa200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xa600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xa600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64d20 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xa600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xaa00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xaa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64d20 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xaa00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xae00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xae00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64d20 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xae00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xb200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64d20 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xb200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xb600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xb600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64d20 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xb600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xba00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xba00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64d20 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xba00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xbe00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xbe00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64d20 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xbe00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xc200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8d90 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xc200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xc600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xc600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8d90 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xc600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xca00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xca00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8d90 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xca00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xce00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xce00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8d90 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xce00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xd200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8d90 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xd200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xd600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xd600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8d90 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xd600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xda00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xda00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8d90 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xda00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xde00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xde00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8d90 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xde00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xe200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9340 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xe200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xe600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xe600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9340 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xe600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xea00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xea00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9340 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xea00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xee00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xee00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9340 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xee00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xf200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9340 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xf200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xf600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xf600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9340 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xf600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xfa00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9340 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xfa00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xfe00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0xfe00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9340 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0xfe00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x10200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9420 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x10200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x10600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9420 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x10600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x10a00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9420 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x10a00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x10e00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x10e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9420 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x10e00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x11200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9420 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x11200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x11600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9420 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x11600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x11a00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9420 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x11a00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x11e00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x11e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9420 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x11e00~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x12200~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12200~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9500 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0x200~200 cache has 0x[200~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x12200~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x12600~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12600~200 size 0x52000 (335872) 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9500 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.847+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x12600~200 = 512 2019-12-07T19:34:04.847+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x12a00~200 2019-12-07T19:34:04.847+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9500 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.848+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x12a00~200 = 512 2019-12-07T19:34:04.848+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x12e00~200 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x12e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9500 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.848+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x12e00~200 = 512 2019-12-07T19:34:04.848+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x13200~200 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13200~200 size 0x52000 (335872) 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9500 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0x1200~200 cache has 0x[1200~200] 2019-12-07T19:34:04.848+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x13200~200 = 512 2019-12-07T19:34:04.848+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x13600~200 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13600~200 size 0x52000 (335872) 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9500 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.848+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x13600~200 = 512 2019-12-07T19:34:04.848+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x13a00~200 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9500 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.848+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x13a00~200 = 512 2019-12-07T19:34:04.848+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x13e00~200 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x13e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9500 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.848+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x13e00~200 = 512 2019-12-07T19:34:04.848+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x14200~200 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14200~200 size 0x52000 (335872) 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f951f0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f951f0 sbid 0x0)) need {<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x14200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.848+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 172 ==== MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:head, version: 27'18, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@27'18, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 263278+0+0 (unknown 0 0 0) 0x558e818ee600 con 0x558e818d7c00 2019-12-07T19:34:04.848+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e81942f20 prio 127 cost 263144 latency 0.000027 epoch 27 MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:head, version: 27'18, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@27'18, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.848+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:head, version: 27'18, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@27'18, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e27) 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:head, version: 27'18, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@27'18, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e27) queued 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:head, version: 27'18, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@27'18, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 263144 e27) pg 0x558e841a4000 2019-12-07T19:34:04.848+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e81942f20 prio 127 cost 263144 latency 0.000072 MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:head, version: 27'18, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@27'18, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 27'23 lc 27'17 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=5 mbc={} ps=[2~1]] 2019-12-07T19:34:04.848+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'17 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=5 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:head, version: 27'18, data_included: [0~262144], data_size: 262144, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@27'18, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'17 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=5 mbc={} ps=[2~1]] get_temp_recovery_object -3:10779340:::temp_recovering_1.8s5_27'18_26_head:head 2019-12-07T19:34:04.848+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'17 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=5 mbc={} ps=[2~1]] handle_recovery_push: Adding oid -3:10779340:::temp_recovering_1.8s5_27'18_26_head:head in the temp collection 2019-12-07T19:34:04.848+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e83e72600 seq 17 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# key 0x857ffffffffffffffd107793'@!temp_recovering_1.8s5_27'18_26_head!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.848+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# key 0x857ffffffffffffffd107793'@!temp_recovering_1.8s5_27'18_26_head!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.848+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1140 2019-12-07T19:34:04.848+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# = 0 2019-12-07T19:34:04.848+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# 0x0~40000 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# 0x0~40000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.848+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~40000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72600 32 blobs 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f951f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x14200:200~200]} 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83f95208 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84670bd0 space 0x558e83f95208 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.848+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate 0x40000/1000,40000,0 2019-12-07T19:34:04.848+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x14200~200 = 512 2019-12-07T19:34:04.848+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x14600~200 2019-12-07T19:34:04.848+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x4da000~40000/1000,40000,0 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14600~200 size 0x52000 (335872) 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x4da000~40000] 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a2cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f61180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f951f0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.848+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x14600~200 = 512 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a2cb0 blob([0x4da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61180 sbid 0x0)) 2019-12-07T19:34:04.848+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x14a00~200 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore.blob(0x558e841a2cb0) get_ref 0x0~2000 Blob(0x558e841a2cb0 blob([0x4da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61180 sbid 0x0)) 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore.blob(0x558e841a2cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f951f0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.848+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x14a00~200 = 512 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e841a2cb0 blob([0x4da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f61180 sbid 0x0)) 2019-12-07T19:34:04.848+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x14e00~200 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x14e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f61198 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f951f0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.848+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x14e00~200 = 512 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f60fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464db20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.848+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x15200~200 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15200~200 size 0x52000 (335872) 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f60fc0 blob([0x4dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464db20 sbid 0x0)) 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f951f0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f60fc0) get_ref 0x0~2000 Blob(0x558e83f60fc0 blob([0x4dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464db20 sbid 0x0)) 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f60fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f951f0 sbid 0x0)) need {<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83f60fc0 blob([0x4dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464db20 sbid 0x0)) 2019-12-07T19:34:04.848+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x15200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464db38 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f60e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456af50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f60e70 blob([0x4de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456af50 sbid 0x0)) 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f60e70) get_ref 0x0~2000 Blob(0x558e83f60e70 blob([0x4de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456af50 sbid 0x0)) 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f60e70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.848+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83f60e70 blob([0x4de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456af50 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456af68 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456afc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456b030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456afc0 blob([0x4e0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b030 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456afc0) get_ref 0x0~2000 Blob(0x558e8456afc0 blob([0x4e0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b030 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456afc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e8456afc0 blob([0x4e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456b030 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456b048 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456b0a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456b110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456b0a0 blob([0x4e2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b110 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b0a0) get_ref 0x0~2000 Blob(0x558e8456b0a0 blob([0x4e2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b110 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e8456b0a0 blob([0x4e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456b110 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456b128 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456b180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456b1f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456b180 blob([0x4e4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b1f0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b180) get_ref 0x0~2000 Blob(0x558e8456b180 blob([0x4e4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b1f0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b180) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e8456b180 blob([0x4e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456b1f0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456b208 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456b260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456b2d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456b260 blob([0x4e6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b2d0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b260) get_ref 0x0~2000 Blob(0x558e8456b260 blob([0x4e6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b2d0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b260) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e8456b260 blob([0x4e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456b2d0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456b2e8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456b340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456b3b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456b340 blob([0x4e8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b3b0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b340) get_ref 0x0~2000 Blob(0x558e8456b340 blob([0x4e8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b3b0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b340) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e8456b340 blob([0x4e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456b3b0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456b3c8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456b420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456b490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456b420 blob([0x4ea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b490 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b420) get_ref 0x0~2000 Blob(0x558e8456b420 blob([0x4ea000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b490 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b420) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e8456b420 blob([0x4ea000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456b490 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456b4a8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456b500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456b570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456b500 blob([0x4ec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b570 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b500) get_ref 0x0~2000 Blob(0x558e8456b500 blob([0x4ec000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b570 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b500) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e8456b500 blob([0x4ec000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456b570 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456b588 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456b5e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456b650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456b5e0 blob([0x4ee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b650 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b5e0) get_ref 0x0~2000 Blob(0x558e8456b5e0 blob([0x4ee000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b650 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e8456b5e0 blob([0x4ee000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456b650 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456b668 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456b6c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456b730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456b6c0 blob([0x4f0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b730 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b6c0) get_ref 0x0~2000 Blob(0x558e8456b6c0 blob([0x4f0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b730 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b6c0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e8456b6c0 blob([0x4f0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456b730 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456b748 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456b7a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84030ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456b7a0 blob([0x4f2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84030ee0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b7a0) get_ref 0x0~2000 Blob(0x558e8456b7a0 blob([0x4f2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84030ee0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b7a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e8456b7a0 blob([0x4f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84030ee0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84030ef8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f60cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a20e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f60cb0 blob([0x4f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a20e0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f60cb0) get_ref 0x0~2000 Blob(0x558e83f60cb0 blob([0x4f4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a20e0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f60cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83f60cb0 blob([0x4f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a20e0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e841a20f8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a2d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f945b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a2d20 blob([0x4f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f945b0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e841a2d20) get_ref 0x0~2000 Blob(0x558e841a2d20 blob([0x4f6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f945b0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e841a2d20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e841a2d20 blob([0x4f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f945b0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f945c8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f95e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83da9f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f95e30 blob([0x4f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da9f10 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f95e30) get_ref 0x0~2000 Blob(0x558e83f95e30 blob([0x4f8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83da9f10 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f95e30) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e83f95e30 blob([0x4f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83da9f10 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83da9f28 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a91b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456b810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a91b90 blob([0x4fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b810 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e81a91b90) get_ref 0x0~2000 Blob(0x558e81a91b90 blob([0x4fa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b810 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e81a91b90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e81a91b90 blob([0x4fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456b810 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456b828 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456b880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456b8f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456b880 blob([0x4fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b8f0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b880) get_ref 0x0~2000 Blob(0x558e8456b880 blob([0x4fc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b8f0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b880) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e8456b880 blob([0x4fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456b8f0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456b908 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456b960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456b9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456b960 blob([0x4fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b9d0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b960) get_ref 0x0~2000 Blob(0x558e8456b960 blob([0x4fe000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456b9d0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456b960) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e8456b960 blob([0x4fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456b9d0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456b9e8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456ba40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456bab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456ba40 blob([0x500000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456bab0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456ba40) get_ref 0x0~2000 Blob(0x558e8456ba40 blob([0x500000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456bab0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456ba40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e8456ba40 blob([0x500000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456bab0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456bac8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456bb20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456bb90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456bb20 blob([0x502000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456bb90 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456bb20) get_ref 0x0~2000 Blob(0x558e8456bb20 blob([0x502000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456bb90 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456bb20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e8456bb20 blob([0x502000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456bb90 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456bba8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456bc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456bc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456bc00 blob([0x504000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456bc70 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456bc00) get_ref 0x0~2000 Blob(0x558e8456bc00 blob([0x504000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456bc70 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456bc00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558e8456bc00 blob([0x504000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456bc70 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456bc88 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456bce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456bd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456bce0 blob([0x506000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456bd50 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456bce0) get_ref 0x0~2000 Blob(0x558e8456bce0 blob([0x506000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456bd50 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456bce0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558e8456bce0 blob([0x506000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456bd50 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456bd68 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456bdc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456be30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456bdc0 blob([0x508000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456be30 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456bdc0) get_ref 0x0~2000 Blob(0x558e8456bdc0 blob([0x508000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456be30 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456bdc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558e8456bdc0 blob([0x508000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456be30 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456be48 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456bea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456bf10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456bea0 blob([0x50a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456bf10 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456bea0) get_ref 0x0~2000 Blob(0x558e8456bea0 blob([0x50a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456bf10 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456bea0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558e8456bea0 blob([0x50a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456bf10 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456bf28 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456bf80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456bf80 blob([0x50c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a000 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456bf80) get_ref 0x0~2000 Blob(0x558e8456bf80 blob([0x50c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a000 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456bf80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558e8456bf80 blob([0x50c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5a000 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f5a018 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5a070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5a070 blob([0x50e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a0e0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5a070) get_ref 0x0~2000 Blob(0x558e83f5a070 blob([0x50e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a0e0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5a070) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558e83f5a070 blob([0x50e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5a0e0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f5a0f8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5a150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5a150 blob([0x510000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a1c0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5a150) get_ref 0x0~2000 Blob(0x558e83f5a150 blob([0x510000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a1c0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5a150) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558e83f5a150 blob([0x510000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5a1c0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f5a1d8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5a230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a2a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5a230 blob([0x512000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a2a0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5a230) get_ref 0x0~2000 Blob(0x558e83f5a230 blob([0x512000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a2a0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5a230) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558e83f5a230 blob([0x512000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5a2a0 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f5a2b8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5a310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5a310 blob([0x514000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a380 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5a310) get_ref 0x0~2000 Blob(0x558e83f5a310 blob([0x514000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a380 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5a310) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558e83f5a310 blob([0x514000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5a380 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f5a398 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5a3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5a3f0 blob([0x516000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a460 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5a3f0) get_ref 0x0~2000 Blob(0x558e83f5a3f0 blob([0x516000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a460 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5a3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558e83f5a3f0 blob([0x516000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5a460 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f5a478 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5a4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5a4d0 blob([0x518000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a540 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5a4d0) get_ref 0x0~2000 Blob(0x558e83f5a4d0 blob([0x518000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5a540 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5a4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558e83f5a4d0 blob([0x518000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5a540 sbid 0x0)) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f5a558 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x40000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e84315788) dirty_range mark inline shard dirty 2019-12-07T19:34:04.849+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# 0x0~40000 = 0 2019-12-07T19:34:04.849+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# 4 keys 2019-12-07T19:34:04.849+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# 4 keys = 0 2019-12-07T19:34:04.849+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72600 cost 395132 (33 ios * 4000 + 263132 bytes) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72600 onodes 0x558e84315680 shared_blobs 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e84315788) update 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e84315788) update inline shard 611 bytes from 32 extents 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# is 1174 (557 bytes onode + 2 bytes spanning blobs + 615 bytes inline extents) 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72600 allocated 0x[4da000~40000] released 0x[] 2019-12-07T19:34:04.849+0000 7fe2c7a66700 10 freelist allocate 0x4da000~40000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 freelist _xor first_key 0x480000 last_key 0x500000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72600 prepare 2019-12-07T19:34:04.849+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83e72600 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f951f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x15200:1200~200]} 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83f95208 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84670c60 space 0x558e83f95208 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x15200~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x15600~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15600~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f951f0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x15600~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x15a00~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f951f0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x15a00~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x15e00~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x15e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95260 blob([0x174000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f951f0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x15e00~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x16200~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16200~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f950a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f950a0 sbid 0x0)) need {<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x16200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f950a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x16200:200~200]} 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83f950b8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84670cf0 space 0x558e83f950b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x16200~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x16600~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16600~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f950a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x16600~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x16a00~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f950a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x16a00~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x16e00~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x16e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f950a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x16e00~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x17200~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17200~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f950a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f950a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x17200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f950a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x17200:1200~200]} 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83f950b8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84670d80 space 0x558e83f950b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x17200~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x17600~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17600~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f950a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x17600~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x17a00~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f950a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x17a00~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x17e00~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x17e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95180 blob([0x176000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f950a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x17e00~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x18200~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18200~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edecb0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edecb0 sbid 0x0)) need {<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x18200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edecb0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x18200:200~200]} 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83edecc8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456f440 space 0x558e83edecc8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x18200~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x18600~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18600~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edecb0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x18600~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x18a00~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edecb0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x18a00~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x18e00~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x18e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edecb0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.849+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x18e00~200 = 512 2019-12-07T19:34:04.849+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x19200~200 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19200~200 size 0x52000 (335872) 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edecb0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edecb0 sbid 0x0)) need {<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x19200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.849+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e81942f20 finish 2019-12-07T19:34:04.849+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.849+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.849+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.850+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72600 aio_wait 2019-12-07T19:34:04.850+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72600 2019-12-07T19:34:04.850+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72600 io_done 2019-12-07T19:34:04.850+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.850+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.850+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edecb0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x19200:1200~200]} 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83edecc8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456f4d0 space 0x558e83edecc8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x19200~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x19600~200 2019-12-07T19:34:04.850+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1382 < min_flush_size 524288 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19600~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edecb0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x19600~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x19a00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edecb0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x19a00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x19e00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x19e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84315680 had 1 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95030 blob([0x273000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edecb0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.850+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1405 < min_flush_size 524288 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x19e00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1405 < min_flush_size 524288 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1a200~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a200~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0bd0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0bd0 sbid 0x0)) need {<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:04.850+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1a200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.850+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xd23c7~57d to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.850+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xd23c7 0xd23c7~57d to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.850+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.850+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa23c7 2019-12-07T19:34:04.850+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x3c7 2019-12-07T19:34:04.850+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.850+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x944 and padding block with 0x6bc 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0bd0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1a200:200~200]} 2019-12-07T19:34:04.850+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xd2944 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e811d0be8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.850+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.850+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456f560 space 0x558e811d0be8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.850+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.850+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1a200~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1a600~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1a600~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000159960s (0.000047182s flush + 0.000112778s kv commit) 2019-12-07T19:34:04.850+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0bd0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e72600> 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.850+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72600 kv_submitted 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1a600~200 = 512 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72600 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1aa00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1aa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72600 finishing 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72600 onodes 0x558e84315680 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0bd0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83da9f28 in 0x558e80219960) _finish_write discard buffer(0x558e8456e510 space 0x558e83da9f28 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1aa00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5a018 in 0x558e80219960) _finish_write discard buffer(0x558e8456eab0 space 0x558e83f5a018 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1ae00~200 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5a0f8 in 0x558e80219960) _finish_write discard buffer(0x558e8456eb40 space 0x558e83f5a0f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ae00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5a1d8 in 0x558e80219960) _finish_write discard buffer(0x558e8456ebd0 space 0x558e83f5a1d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 27/26 [PushReplyOp(1:10779340:::smithi13140796-32:head)]) v3 -- 0x558e81965200 con 0x558e818d7c00 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5a2b8 in 0x558e80219960) _finish_write discard buffer(0x558e8456ec60 space 0x558e83f5a2b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0bd0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5a398 in 0x558e80219960) _finish_write discard buffer(0x558e8456ecf0 space 0x558e83f5a398 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5a478 in 0x558e80219960) _finish_write discard buffer(0x558e8456ed80 space 0x558e83f5a478 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1ae00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5a558 in 0x558e80219960) _finish_write discard buffer(0x558e8456ee10 space 0x558e83f5a558 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1b200~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b200~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f61198 in 0x558e80219960) _finish_write discard buffer(0x558e84671c20 space 0x558e83f61198 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f945c8 in 0x558e80219960) _finish_write discard buffer(0x558e8456e480 space 0x558e83f945c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0bd0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84030ef8 in 0x558e80219960) _finish_write discard buffer(0x558e8456e360 space 0x558e84030ef8 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a20f8 in 0x558e80219960) _finish_write discard buffer(0x558e8456e3f0 space 0x558e841a20f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0bd0 sbid 0x0)) need {<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1b200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456af68 in 0x558e80219960) _finish_write discard buffer(0x558e84671d40 space 0x558e8456af68 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456b048 in 0x558e80219960) _finish_write discard buffer(0x558e84671dd0 space 0x558e8456b048 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456b128 in 0x558e80219960) _finish_write discard buffer(0x558e84671e60 space 0x558e8456b128 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456b208 in 0x558e80219960) _finish_write discard buffer(0x558e84671ef0 space 0x558e8456b208 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456b2e8 in 0x558e80219960) _finish_write discard buffer(0x558e8456e000 space 0x558e8456b2e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456b3c8 in 0x558e80219960) _finish_write discard buffer(0x558e8456e090 space 0x558e8456b3c8 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456b4a8 in 0x558e80219960) _finish_write discard buffer(0x558e8456e120 space 0x558e8456b4a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456b588 in 0x558e80219960) _finish_write discard buffer(0x558e8456e1b0 space 0x558e8456b588 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456b668 in 0x558e80219960) _finish_write discard buffer(0x558e8456e240 space 0x558e8456b668 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456b748 in 0x558e80219960) _finish_write discard buffer(0x558e8456e2d0 space 0x558e8456b748 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456b828 in 0x558e80219960) _finish_write discard buffer(0x558e8456e5a0 space 0x558e8456b828 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456b908 in 0x558e80219960) _finish_write discard buffer(0x558e8456e630 space 0x558e8456b908 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456b9e8 in 0x558e80219960) _finish_write discard buffer(0x558e8456e6c0 space 0x558e8456b9e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456bac8 in 0x558e80219960) _finish_write discard buffer(0x558e8456e750 space 0x558e8456bac8 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456bba8 in 0x558e80219960) _finish_write discard buffer(0x558e8456e7e0 space 0x558e8456bba8 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456bc88 in 0x558e80219960) _finish_write discard buffer(0x558e8456e870 space 0x558e8456bc88 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456bd68 in 0x558e80219960) _finish_write discard buffer(0x558e8456e900 space 0x558e8456bd68 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456be48 in 0x558e80219960) _finish_write discard buffer(0x558e8456e990 space 0x558e8456be48 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456bf28 in 0x558e80219960) _finish_write discard buffer(0x558e8456ea20 space 0x558e8456bf28 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464db38 in 0x558e80219960) _finish_write discard buffer(0x558e84671cb0 space 0x558e8464db38 0x0~2000 writing nocache) 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72600 done 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.850+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72600 [] 2019-12-07T19:34:04.850+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.850+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0bd0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1b200:1200~200]} 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e811d0be8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456f5f0 space 0x558e811d0be8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1b200~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1b600~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1b600~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0bd0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1b600~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1ba00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ba00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0bd0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1ba00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1be00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1be00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83edef50 blob([0x275000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0bd0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1be00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1c200~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c200~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0850 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0850 sbid 0x0)) need {<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1c200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.850+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0850 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1c200:200~200]} 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e811d0868 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456f680 space 0x558e811d0868 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1c200~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1c600~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1c600~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0850 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1c600~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1ca00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ca00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0850 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1ca00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1ce00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ce00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0850 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1ce00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1d200~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d200~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0850 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0850 sbid 0x0)) need {<0x1000, 0x1000> : [0x1d200:1200~200]} 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1d200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0850 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1d200:1200~200]} 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e811d0868 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456f710 space 0x558e811d0868 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1d200~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1d600~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1d600~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0850 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1d600~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1da00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1da00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0850 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1da00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1de00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1de00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e811d0a80 blob([0x277000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d0850 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1de00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1e200~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e200~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5b90 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5b90 sbid 0x0)) need {<0x0, 0x1000> : [0x1e200:200~200]} 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1e200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5b90 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x1e200:200~200]} 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83ac5ba8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456f7a0 space 0x558e83ac5ba8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1e200~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1e600~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1e600~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5b90 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1e600~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1ea00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ea00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5b90 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1ea00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1ee00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1ee00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5b90 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1ee00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1f200~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f200~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5b90 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5b90 sbid 0x0)) need {<0x1000, 0x1000> : [0x1f200:1200~200]} 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1f200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5b90 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1f200:1200~200]} 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83ac5ba8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456f830 space 0x558e83ac5ba8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1f200~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1f600~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1f600~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5b90 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1f600~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1fa00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5b90 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1fa00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1fe00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x1fe00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014310 blob([0x279000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5b90 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x1fe00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x20200~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20200~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e89a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e89a0 sbid 0x0)) need {<0x0, 0x1000> : [0x20200:200~200]} 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x20200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e89a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x20200:200~200]} 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e817e89b8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456f8c0 space 0x558e817e89b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x20200~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x20600~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20600~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e89a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x20600~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x20a00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e89a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x20a00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x20e00~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x20e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e89a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.850+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x20e00~200 = 512 2019-12-07T19:34:04.850+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x21200~200 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21200~200 size 0x52000 (335872) 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e89a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e89a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x21200:1200~200]} 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x21200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.850+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e89a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x21200:1200~200]} 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e817e89b8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456f950 space 0x558e817e89b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x21200~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x21600~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21600~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e89a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x21600~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x21a00~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e89a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x21a00~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x21e00~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x21e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8cb0 blob([0x27b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e89a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x21e00~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x22200~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22200~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e80f04150 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e80f04150 sbid 0x0)) need {<0x0, 0x1000> : [0x22200:200~200]} 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x22200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e80f04150 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x22200:200~200]} 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e80f04168 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456f9e0 space 0x558e80f04168 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x22200~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x22600~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22600~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e80f04150 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x22600~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x22a00~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e80f04150 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x22a00~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x22e00~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x22e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e80f04150 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x22e00~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x23200~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23200~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e80f04150 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e80f04150 sbid 0x0)) need {<0x1000, 0x1000> : [0x23200:1200~200]} 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x23200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e80f04150 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x23200:1200~200]} 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e80f04168 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456fa70 space 0x558e80f04168 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x23200~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x23600~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23600~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e80f04150 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x23600~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x23a00~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e80f04150 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x23a00~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x23e00~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x23e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81867180 blob([0x27d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e80f04150 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x23e00~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x24200~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24200~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90070 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90070 sbid 0x0)) need {<0x0, 0x1000> : [0x24200:200~200]} 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x24200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90070 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x24200:200~200]} 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90088 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456fb00 space 0x558e81a90088 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x24200~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x24600~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24600~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90070 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x24600~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x24a00~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90070 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x24a00~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x24e00~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x24e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90070 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x24e00~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x25200~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25200~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90070 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90070 sbid 0x0)) need {<0x1000, 0x1000> : [0x25200:1200~200]} 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x25200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90070 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x25200:1200~200]} 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90088 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456fb90 space 0x558e81a90088 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x25200~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x25600~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25600~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90070 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x25600~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x25a00~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90070 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x25a00~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x25e00~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x25e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90000 blob([0x27f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90070 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x25e00~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x26200~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26200~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90150 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90150 sbid 0x0)) need {<0x0, 0x1000> : [0x26200:200~200]} 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x26200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90150 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x26200:200~200]} 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90168 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456fc20 space 0x558e81a90168 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x26200~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x26600~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26600~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90150 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x26600~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x26a00~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90150 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x26a00~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x26e00~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x26e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90150 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x26e00~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x27200~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27200~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90150 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90150 sbid 0x0)) need {<0x1000, 0x1000> : [0x27200:1200~200]} 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x27200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90150 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x27200:1200~200]} 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90168 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456fcb0 space 0x558e81a90168 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x27200~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x27600~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27600~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90150 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x27600~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x27a00~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90150 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x27a00~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x27e00~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x27e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a900e0 blob([0x281000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90150 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.851+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x27e00~200 = 512 2019-12-07T19:34:04.851+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x28200~200 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28200~200 size 0x52000 (335872) 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90230 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90230 sbid 0x0)) need {<0x0, 0x1000> : [0x28200:200~200]} 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x28200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.851+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90230 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x28200:200~200]} 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90248 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456fd40 space 0x558e81a90248 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x28200~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x28600~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28600~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90230 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x28600~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x28a00~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90230 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x28a00~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x28e00~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x28e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90230 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x28e00~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x29200~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29200~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90230 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90230 sbid 0x0)) need {<0x1000, 0x1000> : [0x29200:1200~200]} 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x29200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90230 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x29200:1200~200]} 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90248 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456fdd0 space 0x558e81a90248 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x29200~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x29600~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29600~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90230 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x29600~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x29a00~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90230 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x29a00~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x29e00~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x29e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a901c0 blob([0x283000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90230 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x29e00~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2a200~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a200~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90310 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90310 sbid 0x0)) need {<0x0, 0x1000> : [0x2a200:200~200]} 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2a200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90310 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2a200:200~200]} 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90328 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456fe60 space 0x558e81a90328 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2a200~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2a600~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2a600~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90310 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2a600~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2aa00~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2aa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90310 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2aa00~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2ae00~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ae00~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90310 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2ae00~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2b200~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b200~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90310 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90310 sbid 0x0)) need {<0x1000, 0x1000> : [0x2b200:1200~200]} 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2b200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90310 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2b200:1200~200]} 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90328 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e8456fef0 space 0x558e81a90328 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2b200~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2b600~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2b600~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90310 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2b600~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2ba00~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ba00~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90310 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2ba00~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2be00~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2be00~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a902a0 blob([0x285000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90310 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2be00~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2c200~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c200~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a903f0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a903f0 sbid 0x0)) need {<0x0, 0x1000> : [0x2c200:200~200]} 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2c200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a903f0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2c200:200~200]} 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90408 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84596000 space 0x558e81a90408 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2c200~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2c600~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2c600~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a903f0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2c600~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2ca00~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ca00~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a903f0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2ca00~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2ce00~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ce00~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a903f0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2ce00~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2d200~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d200~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a903f0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a903f0 sbid 0x0)) need {<0x1000, 0x1000> : [0x2d200:1200~200]} 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2d200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a903f0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2d200:1200~200]} 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90408 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84596090 space 0x558e81a90408 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2d200~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2d600~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2d600~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a903f0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2d600~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2da00~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2da00~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a903f0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2da00~200 = 512 2019-12-07T19:34:04.852+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2de00~200 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2de00~200 size 0x52000 (335872) 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.852+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90380 blob([0x287000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a903f0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.852+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2de00~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2e200~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e200~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a904d0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a904d0 sbid 0x0)) need {<0x0, 0x1000> : [0x2e200:200~200]} 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2e200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a904d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x2e200:200~200]} 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a904e8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84596120 space 0x558e81a904e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2e200~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2e600~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2e600~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a904d0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2e600~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2ea00~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ea00~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a904d0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2ea00~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2ee00~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2ee00~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a904d0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2ee00~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2f200~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f200~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a904d0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a904d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x2f200:1200~200]} 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2f200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a904d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x2f200:1200~200]} 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a904e8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e845961b0 space 0x558e81a904e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2f200~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2f600~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2f600~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a904d0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2f600~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2fa00~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a904d0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2fa00~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2fe00~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x2fe00~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90460 blob([0x289000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a904d0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x2fe00~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x30200~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30200~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a905b0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a905b0 sbid 0x0)) need {<0x0, 0x1000> : [0x30200:200~200]} 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x30200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a905b0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x30200:200~200]} 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a905c8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84596240 space 0x558e81a905c8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x30200~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x30600~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30600~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a905b0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x30600~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x30a00~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a905b0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x30a00~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x30e00~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x30e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a905b0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x30e00~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x31200~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31200~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a905b0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a905b0 sbid 0x0)) need {<0x1000, 0x1000> : [0x31200:1200~200]} 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x31200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a905b0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x31200:1200~200]} 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a905c8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e845962d0 space 0x558e81a905c8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x31200~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x31600~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31600~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a905b0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x31600~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x31a00~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a905b0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x31a00~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x31e00~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x31e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90540 blob([0x28b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a905b0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x31e00~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x32200~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32200~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90690 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90690 sbid 0x0)) need {<0x0, 0x1000> : [0x32200:200~200]} 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x32200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90690 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x32200:200~200]} 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a906a8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84596360 space 0x558e81a906a8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x32200~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x32600~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32600~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90690 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x32600~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x32a00~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90690 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x32a00~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x32e00~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x32e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90690 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x32e00~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x33200~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33200~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90690 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90690 sbid 0x0)) need {<0x1000, 0x1000> : [0x33200:1200~200]} 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x33200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90690 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x33200:1200~200]} 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a906a8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e845963f0 space 0x558e81a906a8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x33200~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x33600~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33600~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90690 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x33600~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x33a00~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90690 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x33a00~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x33e00~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x33e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90620 blob([0x28d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90690 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x33e00~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x34200~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34200~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90770 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90770 sbid 0x0)) need {<0x0, 0x1000> : [0x34200:200~200]} 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x34200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90770 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x34200:200~200]} 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90788 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84596480 space 0x558e81a90788 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.853+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x34200~200 = 512 2019-12-07T19:34:04.853+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x34600~200 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34600~200 size 0x52000 (335872) 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.853+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90770 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x34600~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x34a00~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90770 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x34a00~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x34e00~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x34e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90770 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x34e00~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x35200~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35200~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90770 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90770 sbid 0x0)) need {<0x1000, 0x1000> : [0x35200:1200~200]} 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x35200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90770 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x35200:1200~200]} 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90788 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84596510 space 0x558e81a90788 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x35200~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x35600~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35600~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90770 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x35600~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x35a00~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90770 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x35a00~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x35e00~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x35e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a90700 blob([0x28f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90770 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x35e00~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x36200~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36200~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90850 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90850 sbid 0x0)) need {<0x0, 0x1000> : [0x36200:200~200]} 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x36200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90850 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x36200:200~200]} 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90868 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e845965a0 space 0x558e81a90868 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x36200~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x36600~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36600~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90850 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x36600~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x36a00~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90850 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x36a00~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x36e00~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x36e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90850 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x36e00~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x37200~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37200~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90850 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90850 sbid 0x0)) need {<0x1000, 0x1000> : [0x37200:1200~200]} 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x37200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90850 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x37200:1200~200]} 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90868 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84596630 space 0x558e81a90868 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x37200~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x37600~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37600~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90850 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x37600~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x37a00~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90850 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x37a00~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x37e00~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x37e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a907e0 blob([0x291000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90850 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x37e00~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x38200~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38200~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90930 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90930 sbid 0x0)) need {<0x0, 0x1000> : [0x38200:200~200]} 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x38200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90930 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x38200:200~200]} 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90948 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e845966c0 space 0x558e81a90948 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x38200~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x38600~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38600~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90930 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x38600~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x38a00~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90930 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x38a00~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x38e00~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x38e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90930 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x38e00~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x39200~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39200~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90930 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90930 sbid 0x0)) need {<0x1000, 0x1000> : [0x39200:1200~200]} 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x39200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90930 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x39200:1200~200]} 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90948 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84596750 space 0x558e81a90948 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x39200~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x39600~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39600~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90930 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x39600~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x39a00~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90930 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x39a00~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x39e00~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x39e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a908c0 blob([0x293000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a90930 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x39e00~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3a200~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a200~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a909a0 blob([0x295000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e81a90a10 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a909a0 blob([0x295000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e81a90a10 sbid 0x0)) need {<0x0, 0x1000> : [0x3a200:200~200]} 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3a200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a909a0 blob([0x295000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e81a90a10 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3a200:200~200]} 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e81a90a28 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e845967e0 space 0x558e81a90a28 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3a200~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3a600~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3a600~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a909a0 blob([0x295000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e81a90a10 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3a600~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3aa00~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3aa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a909a0 blob([0x295000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e81a90a10 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3aa00~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3ae00~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ae00~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a909a0 blob([0x295000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e81a90a10 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.854+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3ae00~200 = 512 2019-12-07T19:34:04.854+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3b200~200 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b200~200 size 0x52000 (335872) 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f956c0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f956c0 sbid 0x0)) need {<0x0, 0x1000> : [0x3b200:200~200]} 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3b200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.854+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f956c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3b200:200~200]} 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83f956d8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84596870 space 0x558e83f956d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3b200~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3b600~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3b600~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f956c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3b600~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3ba00~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ba00~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f956c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3ba00~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3be00~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3be00~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f956c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3be00~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3c200~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c200~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f956c0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f956c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x3c200:1200~200]} 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3c200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f956c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3c200:1200~200]} 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83f956d8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84596900 space 0x558e83f956d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3c200~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3c600~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3c600~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f956c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3c600~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3ca00~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ca00~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f956c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3ca00~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3ce00~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ce00~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28850 blob([0x336000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f956c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3ce00~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3d200~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d200~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b289a0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b289a0 sbid 0x0)) need {<0x0, 0x1000> : [0x3d200:200~200]} 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3d200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b289a0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3d200:200~200]} 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83b289b8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84596990 space 0x558e83b289b8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3d200~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3d600~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3d600~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b289a0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3d600~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3da00~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3da00~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b289a0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3da00~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3de00~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3de00~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b289a0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3de00~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3e200~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e200~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b289a0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b289a0 sbid 0x0)) need {<0x1000, 0x1000> : [0x3e200:1200~200]} 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3e200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b289a0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x3e200:1200~200]} 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83b289b8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84596a20 space 0x558e83b289b8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3e200~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3e600~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3e600~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b289a0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3e600~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3ea00~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ea00~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b289a0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3ea00~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3ee00~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3ee00~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28690 blob([0x338000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b289a0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3ee00~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3f200~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f200~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b281c0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b281c0 sbid 0x0)) need {<0x0, 0x1000> : [0x3f200:200~200]} 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x3f200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b281c0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x3f200:200~200]} 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore.BufferSpace(0x558e83b281d8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 _add level 1 near 0 on buffer(0x558e84596ab0 space 0x558e83b281d8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3f200~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3f600~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3f600~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b281c0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3f600~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3fa00~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b281c0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3fa00~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3fe00~200 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x3fe00~200 size 0x52000 (335872) 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b281c0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x3fe00~200 = 512 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=512 len=131072 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:f7919c75:::smithi13140796-20:head 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:f7919c75:::smithi13140796-20:head 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 335872 2019-12-07T19:34:04.855+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key = 0 2019-12-07T19:34:04.855+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.fs0 27/26 ECSubReadReply(tid=82, attrs_read=0)) v2 -- 0x558e811dc000 con 0x558e818d7000 2019-12-07T19:34:04.855+0000 7fe2c8267700 10 osd.0 27 dequeue_op 0x558e81943600 finish 2019-12-07T19:34:04.855+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.856+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 528522 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.858+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 173 ==== MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:head, version: 27'18, data_included: [262144~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@27'18, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1523712, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 119384+0+0 (unknown 0 0 0) 0x558e811f6400 con 0x558e818d7c00 2019-12-07T19:34:04.858+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edd760 prio 127 cost 119784 latency 0.000030 epoch 27 MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:head, version: 27'18, data_included: [262144~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@27'18, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1523712, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.858+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:head, version: 27'18, data_included: [262144~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@27'18, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1523712, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 119784 e27) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:head, version: 27'18, data_included: [262144~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@27'18, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1523712, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 119784 e27) queued 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:head, version: 27'18, data_included: [262144~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@27'18, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1523712, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 119784 e27) pg 0x558e841a4000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e83edd760 prio 127 cost 119784 latency 0.000083 MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:head, version: 27'18, data_included: [262144~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@27'18, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1523712, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 27'23 lc 27'17 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=5 mbc={} ps=[2~1]] 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'17 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=5 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.8s5 27/26 [PushOp(1:10779340:::smithi13140796-32:head, version: 27'18, data_included: [262144~118784], data_size: 118784, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(1:10779340:::smithi13140796-32:head@27'18, size: 1523712, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1523712, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'17 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=5 mbc={} ps=[2~1]] get_temp_recovery_object -3:10779340:::temp_recovering_1.8s5_27'18_26_head:head 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'17 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=5 mbc={} ps=[2~1]] handle_recovery_push: Removing oid -3:10779340:::temp_recovering_1.8s5_27'18_26_head:head from the temp collection 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'17 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=5 mbc={} ps=[2~1]] on_local_recover: 1:10779340:::smithi13140796-32:head 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'17 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=5 mbc={} ps=[2~1]] got missing 1:10779340:::smithi13140796-32:head v 27'18 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'19 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'17 active m=4 mbc={} ps=[2~1]] last_complete now 27'19 log.complete_to 27'20 2019-12-07T19:34:04.858+0000 7fe2cba6e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e81a0a600 seq 18 2019-12-07T19:34:04.858+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# 0x40000~1d000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# 0x40000~1d000 - have 0x40000 (262144) bytes fadvise_flags 0x0 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x40000~1d000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e81a0a600 15 blobs 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 fbmap_alloc 0x558e80233300 allocate 0x1d000/1000,1d000,0 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x51a000~1d000/1000,1d000,0 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x51a000~1d000] 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f60d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f60d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f60d20 blob([0x51a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f60d90 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f60d20) get_ref 0x0~2000 Blob(0x558e83f60d20 blob([0x51a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f60d90 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f60d20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558e83f60d20 blob([0x51a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f60d90 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f60da8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f60e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a2a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f60e00 blob([0x51c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a2a80 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f60e00) get_ref 0x0~2000 Blob(0x558e83f60e00 blob([0x51c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a2a80 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f60e00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558e83f60e00 blob([0x51c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2a80 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e841a2a98 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f61e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f61ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f61e30 blob([0x51e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61ea0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f61e30) get_ref 0x0~2000 Blob(0x558e83f61e30 blob([0x51e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61ea0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f61e30) get_ref init 0x2000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558e83f61e30 blob([0x51e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f61ea0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f61eb8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5b570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5b570 blob([0x520000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b5e0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5b570) get_ref 0x0~2000 Blob(0x558e83f5b570 blob([0x520000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b5e0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5b570) get_ref init 0x2000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558e83f5b570 blob([0x520000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5b5e0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f5b5f8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5b650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5b650 blob([0x522000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b6c0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5b650) get_ref 0x0~2000 Blob(0x558e83f5b650 blob([0x522000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b6c0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5b650) get_ref init 0x2000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558e83f5b650 blob([0x522000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5b6c0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f5b6d8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5b730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5b730 blob([0x524000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b7a0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5b730) get_ref 0x0~2000 Blob(0x558e83f5b730 blob([0x524000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b7a0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5b730) get_ref init 0x2000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558e83f5b730 blob([0x524000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5b7a0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f5b7b8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5b810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5b810 blob([0x526000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b880 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5b810) get_ref 0x0~2000 Blob(0x558e83f5b810 blob([0x526000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b880 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5b810) get_ref init 0x2000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4c000~2000: 0x0~2000 Blob(0x558e83f5b810 blob([0x526000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5b880 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f5b898 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5b8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5b8f0 blob([0x528000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b960 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5b8f0) get_ref 0x0~2000 Blob(0x558e83f5b8f0 blob([0x528000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b960 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5b8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4e000~2000: 0x0~2000 Blob(0x558e83f5b8f0 blob([0x528000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5b960 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f5b978 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5b9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5ba40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5b9d0 blob([0x52a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5ba40 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5b9d0) get_ref 0x0~2000 Blob(0x558e83f5b9d0 blob([0x52a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5ba40 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5b9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x50000~2000: 0x0~2000 Blob(0x558e83f5b9d0 blob([0x52a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5ba40 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f5ba58 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5bab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5bab0 blob([0x52c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bb20 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5bab0) get_ref 0x0~2000 Blob(0x558e83f5bab0 blob([0x52c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bb20 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5bab0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x52000~2000: 0x0~2000 Blob(0x558e83f5bab0 blob([0x52c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5bb20 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f5bb38 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5bb90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5bb90 blob([0x52e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bc00 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5bb90) get_ref 0x0~2000 Blob(0x558e83f5bb90 blob([0x52e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bc00 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5bb90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x54000~2000: 0x0~2000 Blob(0x558e83f5bb90 blob([0x52e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5bc00 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f5bc18 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5bc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5bc70 blob([0x530000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bce0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5bc70) get_ref 0x0~2000 Blob(0x558e83f5bc70 blob([0x530000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bce0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5bc70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x56000~2000: 0x0~2000 Blob(0x558e83f5bc70 blob([0x530000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5bce0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f5bcf8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5bd50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bdc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5bd50 blob([0x532000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bdc0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5bd50) get_ref 0x0~2000 Blob(0x558e83f5bd50 blob([0x532000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bdc0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5bd50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x58000~2000: 0x0~2000 Blob(0x558e83f5bd50 blob([0x532000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5bdc0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f5bdd8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5be30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5be30 blob([0x534000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bea0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5be30) get_ref 0x0~2000 Blob(0x558e83f5be30 blob([0x534000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bea0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5be30) get_ref init 0x2000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x5a000~2000: 0x0~2000 Blob(0x558e83f5be30 blob([0x534000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5bea0 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f5beb8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5bf10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bf80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5bf10 blob([0x536000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bf80 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5bf10) get_ref 0x0~1000 Blob(0x558e83f5bf10 blob([0x536000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5bf80 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.blob(0x558e83f5bf10) get_ref init 0x1000, 1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x5c000~1000: 0x0~1000 Blob(0x558e83f5bf10 blob([0x536000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f5bf80 sbid 0x0)) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83f5bf98 in 0x558e80219960) _discard 0x0~1000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x5d000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e84315788) dirty_range mark inline shard dirty 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# 0x40000~1d000 = 0 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:10779340:::smithi13140796-32:head# key 0x858000000000000001107793'@!smithi13140796-32!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:10779340:::smithi13140796-32:head# key 0x858000000000000001107793'@!smithi13140796-32!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.858+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# -> 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.8s5_head 5#-3:10779340:::temp_recovering_1.8s5_27'18_26_head:head# -> 5#1:10779340:::smithi13140796-32:head# = 0 2019-12-07T19:34:04.858+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x000000000000045f'.missing/0000000000000001.80EE9C20.head.smithi13140796-32..' <- missing/0000000000000001.80EE9C20.head.smithi13140796-32.. 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.858+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a600 cost 184689 (16 ios * 4000 + 120689 bytes) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a600 onodes 0x558e84315680 shared_blobs 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e84315788) update 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e84315788) update inline shard 893 bytes from 47 extents 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:10779340:::smithi13140796-32:head# is 1456 (557 bytes onode + 2 bytes spanning blobs + 897 bytes inline extents) 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a600 allocated 0x[51a000~1d000] released 0x[] 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 freelist allocate 0x51a000~1d000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 freelist _xor first_key 0x500000 last_key 0x500000 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 prepare 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e81a0a600 2019-12-07T19:34:04.858+0000 7fe2cba6e700 10 osd.0 27 dequeue_op 0x558e83edd760 finish 2019-12-07T19:34:04.858+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'19 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'17 active m=4 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.858+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.859+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 aio_wait 2019-12-07T19:34:04.859+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a600 2019-12-07T19:34:04.859+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 io_done 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.859+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2969 < min_flush_size 524288 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84315680 had 1 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402e780 had 1 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.859+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2992 < min_flush_size 524288 2019-12-07T19:34:04.859+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2992 < min_flush_size 524288 2019-12-07T19:34:04.859+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.859+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xd2944~bb0 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.859+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xd2944 0xd2944~bb0 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa2944 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x944 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x4f4 and padding block with 0xb0c 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xd34f4 2019-12-07T19:34:04.859+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.859+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.859+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000152473s (0.000045681s flush + 0.000106792s kv commit) 2019-12-07T19:34:04.859+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0a600> 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.859+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 kv_submitted 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a600 2019-12-07T19:34:04.859+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 finishing 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a600 onodes 0x558e84315680 2019-12-07T19:34:04.859+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5b5f8 in 0x558e80219960) _finish_write discard buffer(0x558e845974d0 space 0x558e83f5b5f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5b6d8 in 0x558e80219960) _finish_write discard buffer(0x558e84597560 space 0x558e83f5b6d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5b7b8 in 0x558e80219960) _finish_write discard buffer(0x558e845975f0 space 0x558e83f5b7b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5b898 in 0x558e80219960) _finish_write discard buffer(0x558e84597680 space 0x558e83f5b898 0x0~2000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5b978 in 0x558e80219960) _finish_write discard buffer(0x558e84597710 space 0x558e83f5b978 0x0~2000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5ba58 in 0x558e80219960) _finish_write discard buffer(0x558e845977a0 space 0x558e83f5ba58 0x0~2000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5bb38 in 0x558e80219960) _finish_write discard buffer(0x558e84597830 space 0x558e83f5bb38 0x0~2000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5bc18 in 0x558e80219960) _finish_write discard buffer(0x558e845978c0 space 0x558e83f5bc18 0x0~2000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5bcf8 in 0x558e80219960) _finish_write discard buffer(0x558e84597950 space 0x558e83f5bcf8 0x0~2000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5bdd8 in 0x558e80219960) _finish_write discard buffer(0x558e845979e0 space 0x558e83f5bdd8 0x0~2000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5beb8 in 0x558e80219960) _finish_write discard buffer(0x558e84597a70 space 0x558e83f5beb8 0x0~2000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5bf98 in 0x558e80219960) _finish_write discard buffer(0x558e84597b00 space 0x558e83f5bf98 0x0~1000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'19 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'17 active m=4 mbc={} ps=[2~1]] recovery_committed_to version 27'19 now ondisk 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f60da8 in 0x558e80219960) _finish_write discard buffer(0x558e802299e0 space 0x558e83f60da8 0x0~2000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f61eb8 in 0x558e80219960) _finish_write discard buffer(0x558e84258e10 space 0x558e83f61eb8 0x0~2000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a2a98 in 0x558e80219960) _finish_write discard buffer(0x558e84258d80 space 0x558e841a2a98 0x0~2000 writing nocache) 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a600 done 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.859+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a600 [] 2019-12-07T19:34:04.859+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.859+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.859+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 27/26 [PushReplyOp(1:10779340:::smithi13140796-32:head)]) v3 -- 0x558e81239800 con 0x558e818d7c00 2019-12-07T19:34:04.859+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.860+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 99 ==== MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=83, to_read={1:f7919c75:::smithi13140796-20:head=262144,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 ==== 265+0+0 (unknown 0 0 0) 0x558e811dc000 con 0x558e818d7000 2019-12-07T19:34:04.860+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edcc60 prio 127 cost 0 latency 0.000030 epoch 27 MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=83, to_read={1:f7919c75:::smithi13140796-20:head=262144,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.860+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=83, to_read={1:f7919c75:::smithi13140796-20:head=262144,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 127 cost 0 e27) 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=83, to_read={1:f7919c75:::smithi13140796-20:head=262144,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 127 cost 0 e27) queued 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=83, to_read={1:f7919c75:::smithi13140796-20:head=262144,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3) prio 127 cost 0 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.861+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e83edcc60 prio 127 cost 0 latency 0.000973 MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=83, to_read={1:f7919c75:::smithi13140796-20:head=262144,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 pg pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.861+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.fs1 27/26 ECSubRead(tid=83, to_read={1:f7919c75:::smithi13140796-20:head=262144,262144,0}, subchunks={1:f7919c75:::smithi13140796-20:head=[1,1,3,1,5,1,7,1]}, attrs_to_read=)) v3 2019-12-07T19:34:04.861+0000 7fe2c425f700 25 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] handle_sub_read case2: going to do fragmented read. 2019-12-07T19:34:04.861+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x40200~200 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x40200~200 size 0x52000 (335872) 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b281c0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b281c0 sbid 0x0)) need {<0x1000, 0x1000> : [0x40200:1200~200]} 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x40200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b281c0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x40200:1200~200]} 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83b281d8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84671b90 space 0x558e83b281d8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.861+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x40200~200 = 512 2019-12-07T19:34:04.861+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x40600~200 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x40600~200 size 0x52000 (335872) 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b281c0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.861+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x40600~200 = 512 2019-12-07T19:34:04.861+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x40a00~200 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x40a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b281c0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.861+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x40a00~200 = 512 2019-12-07T19:34:04.861+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x40e00~200 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x40e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28a80 blob([0x33a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b281c0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.861+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x40e00~200 = 512 2019-12-07T19:34:04.861+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x41200~200 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x41200~200 size 0x52000 (335872) 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28150 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28150 sbid 0x0)) need {<0x0, 0x1000> : [0x41200:200~200]} 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x41200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28150 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x41200:200~200]} 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83b28168 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e84597ef0 space 0x558e83b28168 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.861+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x41200~200 = 512 2019-12-07T19:34:04.861+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x41600~200 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x41600~200 size 0x52000 (335872) 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28150 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.861+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x41600~200 = 512 2019-12-07T19:34:04.861+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x41a00~200 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x41a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28150 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.861+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x41a00~200 = 512 2019-12-07T19:34:04.861+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x41e00~200 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x41e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28150 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.861+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x41e00~200 = 512 2019-12-07T19:34:04.861+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x42200~200 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x42200~200 size 0x52000 (335872) 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28150 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28150 sbid 0x0)) need {<0x1000, 0x1000> : [0x42200:1200~200]} 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x42200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.861+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28150 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x42200:1200~200]} 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83b28168 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b8000 space 0x558e83b28168 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x42200~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x42600~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x42600~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28150 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x42600~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x42a00~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x42a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28150 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x42a00~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x42e00~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x42e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b287e0 blob([0x33c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28150 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x42e00~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x43200~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x43200~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b280e0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b280e0 sbid 0x0)) need {<0x0, 0x1000> : [0x43200:200~200]} 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x43200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b280e0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x43200:200~200]} 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83b280f8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b8090 space 0x558e83b280f8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x43200~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x43600~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x43600~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b280e0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x43600~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x43a00~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x43a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b280e0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x43a00~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x43e00~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x43e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b280e0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x43e00~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x44200~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x44200~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b280e0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b280e0 sbid 0x0)) need {<0x1000, 0x1000> : [0x44200:1200~200]} 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x44200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b280e0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x44200:1200~200]} 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83b280f8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b8120 space 0x558e83b280f8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x44200~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x44600~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x44600~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b280e0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x44600~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x44a00~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x44a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b280e0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x44a00~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x44e00~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x44e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28770 blob([0x33e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b280e0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x44e00~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x45200~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x45200~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28070 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28070 sbid 0x0)) need {<0x0, 0x1000> : [0x45200:200~200]} 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x45200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28070 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x45200:200~200]} 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83b28088 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b81b0 space 0x558e83b28088 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x45200~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x45600~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x45600~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28070 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x45600~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x45a00~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x45a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28070 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x45a00~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x45e00~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x45e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28070 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x45e00~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x46200~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x46200~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28070 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28070 sbid 0x0)) need {<0x1000, 0x1000> : [0x46200:1200~200]} 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x46200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28070 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x46200:1200~200]} 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83b28088 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b8240 space 0x558e83b28088 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x46200~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x46600~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x46600~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28070 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x46600~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x46a00~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x46a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28070 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x46a00~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x46e00~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x46e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83b28700 blob([0x340000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b28070 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x46e00~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x47200~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x47200~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48af0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48af0 sbid 0x0)) need {<0x0, 0x1000> : [0x47200:200~200]} 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x47200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48af0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x47200:200~200]} 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83f48b08 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b82d0 space 0x558e83f48b08 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x47200~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x47600~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x47600~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48af0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x47600~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x47a00~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x47a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48af0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x47a00~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x47e00~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x47e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48af0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.862+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x47e00~200 = 512 2019-12-07T19:34:04.862+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x48200~200 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x48200~200 size 0x52000 (335872) 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48af0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48af0 sbid 0x0)) need {<0x1000, 0x1000> : [0x48200:1200~200]} 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x48200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.862+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48af0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x48200:1200~200]} 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83f48b08 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b8360 space 0x558e83f48b08 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x48200~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x48600~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x48600~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48af0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x48600~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x48a00~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x48a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48af0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x48a00~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x48e00~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x48e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48c40 blob([0x342000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48af0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x48e00~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x49200~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x49200~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03f10 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03f10 sbid 0x0)) need {<0x0, 0x1000> : [0x49200:200~200]} 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x49200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03f10 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x49200:200~200]} 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03f28 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b83f0 space 0x558e83d03f28 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x49200~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x49600~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x49600~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03f10 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x49600~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x49a00~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x49a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03f10 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x49a00~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x49e00~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x49e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03f10 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x49e00~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4a200~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4a200~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03f10 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03f10 sbid 0x0)) need {<0x1000, 0x1000> : [0x4a200:1200~200]} 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4a200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03f10 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x4a200:1200~200]} 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d03f28 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b8480 space 0x558e83d03f28 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4a200~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4a600~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4a600~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03f10 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4a600~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4aa00~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4aa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03f10 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4aa00~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4ae00~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4ae00~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f48d90 blob([0x344000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03f10 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4ae00~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4b200~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4b200~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02b60 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02b60 sbid 0x0)) need {<0x0, 0x1000> : [0x4b200:200~200]} 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4b200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02b60 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4b200:200~200]} 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d02b78 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b8510 space 0x558e83d02b78 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4b200~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4b600~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4b600~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02b60 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4b600~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4ba00~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4ba00~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02b60 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4ba00~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4be00~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4be00~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02b60 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4be00~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4c200~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4c200~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02b60 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02b60 sbid 0x0)) need {<0x1000, 0x1000> : [0x4c200:1200~200]} 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4c200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02b60 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x4c200:1200~200]} 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d02b78 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b85a0 space 0x558e83d02b78 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4c200~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4c600~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4c600~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02b60 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4c600~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4ca00~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4ca00~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02b60 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4ca00~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4ce00~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4ce00~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03ea0 blob([0x346000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02b60 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4ce00~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4d200~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4d200~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f340 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f340 sbid 0x0)) need {<0x0, 0x1000> : [0x4d200:200~200]} 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4d200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f340 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4d200:200~200]} 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d0f358 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b8630 space 0x558e83d0f358 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4d200~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4d600~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4d600~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f340 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4d600~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4da00~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4da00~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f340 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4da00~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4de00~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4de00~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f340 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4de00~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4e200~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4e200~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f340 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f340 sbid 0x0)) need {<0x1000, 0x1000> : [0x4e200:1200~200]} 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4e200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f340 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x4e200:1200~200]} 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d0f358 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b86c0 space 0x558e83d0f358 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4e200~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4e600~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4e600~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f340 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4e600~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4ea00~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4ea00~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f340 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4ea00~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4ee00~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4ee00~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03e30 blob([0x348000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f340 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.863+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4ee00~200 = 512 2019-12-07T19:34:04.863+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4f200~200 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4f200~200 size 0x52000 (335872) 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) need 0x200~200 cache has 0x[] 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) need {<0x0, 0x1000> : [0x4f200:200~200]} 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4f200: 0x200 reading 0x0~1000 2019-12-07T19:34:04.863+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) need 0x{<0x0, 0x1000> : [0x4f200:200~200]} 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d0f2e8 in 0x558e8102e000) _discard 0x0~1000 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b8750 space 0x558e83d0f2e8 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4f200~200 = 512 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4f600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4f600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) need 0x600~200 cache has 0x[600~200] 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4f600~200 = 512 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4fa00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4fa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) need 0xa00~200 cache has 0x[a00~200] 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4fa00~200 = 512 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4fe00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x4fe00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) need 0xe00~200 cache has 0x[e00~200] 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x4fe00~200 = 512 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x50200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x50200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) need {<0x1000, 0x1000> : [0x50200:1200~200]} 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x50200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x50200:1200~200]} 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83d0f2e8 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b87e0 space 0x558e83d0f2e8 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x50200~200 = 512 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x50600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x50600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x50600~200 = 512 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x50a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x50a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x50a00~200 = 512 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x50e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x50e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0f260 blob([0x34a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f2d0 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x50e00~200 = 512 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x51200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x51200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0f960 blob([!~1000,0x34c000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558e83af8d20 sbid 0x0)) need 0x1200~200 cache has 0x[] 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83d0f960 blob([!~1000,0x34c000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558e83af8d20 sbid 0x0)) need {<0x1000, 0x1000> : [0x51200:1200~200]} 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x51200: 0x1200 reading 0x1000~1000 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83d0f960 blob([!~1000,0x34c000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558e83af8d20 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x51200:1200~200]} 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore.BufferSpace(0x558e83af8d38 in 0x558e8102e000) _discard 0x1000~1000 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 _add level 1 near 0 on buffer(0x558e842b8870 space 0x558e83af8d38 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x51200~200 = 512 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x51600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x51600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0f960 blob([!~1000,0x34c000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558e83af8d20 sbid 0x0)) need 0x1600~200 cache has 0x[1600~200] 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x51600~200 = 512 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x51a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x51a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0f960 blob([!~1000,0x34c000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558e83af8d20 sbid 0x0)) need 0x1a00~200 cache has 0x[1a00~200] 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x51a00~200 = 512 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x51e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x51e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d0f960 blob([!~1000,0x34c000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558e83af8d20 sbid 0x0)) need 0x1e00~200 cache has 0x[1e00~200] 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x51e00~200 = 512 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x52200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x52200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x52200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x52600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x52600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x52600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x52a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x52a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x52a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x52e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x52e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x52e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x53200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x53200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x53200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x53600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x53600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x53600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x53a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x53a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x53a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x53e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x53e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x53e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x54200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x54200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x54200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x54600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x54600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x54600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x54a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x54a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x54a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x54e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x54e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x54e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x55200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x55200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x55200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x55600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x55600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x55600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x55a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x55a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x55a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x55e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x55e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x55e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x56200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x56200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x56200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x56600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x56600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x56600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x56a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x56a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x56a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x56e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x56e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x56e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x57200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x57200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x57200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x57600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x57600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x57600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x57a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x57a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x57a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x57e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x57e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x57e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x58200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x58200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x58200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x58600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x58600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x58600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x58a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x58a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x58a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x58e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x58e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x58e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x59200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x59200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x59200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x59600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x59600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x59600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x59a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x59a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x59a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x59e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x59e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x59e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5a200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5a200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5a200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5a600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5a600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5a600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5aa00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5aa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5aa00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5ae00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5ae00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5ae00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5b200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5b200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5b200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5b600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5b600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5b600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5ba00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5ba00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5ba00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5be00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5be00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5be00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5c200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5c200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5c200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5c600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5c600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5c600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5ca00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5ca00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5ca00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5ce00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5ce00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5ce00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5d200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5d200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5d200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5d600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5d600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5d600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5da00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5da00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5da00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5de00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5de00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5de00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5e200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5e200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5e200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5e600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5e600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5e600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5ea00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5ea00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5ea00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5ee00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5ee00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5ee00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5f200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5f200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5f200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5f600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5f600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5f600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5fa00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5fa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5fa00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5fe00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x5fe00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x5fe00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x60200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x60200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x60200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x60600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x60600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x60600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x60a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x60a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x60a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x60e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x60e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x60e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x61200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x61200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x61200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x61600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x61600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x61600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x61a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x61a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x61a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x61e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x61e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x61e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x62200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x62200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x62200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x62600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x62600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x62600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x62a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x62a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x62a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x62e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x62e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x62e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x63200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x63200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x63200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x63600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x63600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x63600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x63a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x63a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x63a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x63e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x63e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x63e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x64200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x64200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x64200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x64600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x64600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x64600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x64a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x64a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x64a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x64e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x64e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x64e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x65200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x65200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x65200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x65600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x65600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x65600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x65a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x65a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x65a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x65e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x65e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x65e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x66200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x66200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x66200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x66600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x66600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x66600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x66a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x66a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x66a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x66e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x66e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x66e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x67200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x67200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x67200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x67600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x67600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x67600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x67a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x67a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x67a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x67e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x67e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x67e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x68200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x68200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x68200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x68600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x68600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x68600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x68a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x68a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x68a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x68e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x68e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x68e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x69200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x69200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x69200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x69600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x69600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x69600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x69a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x69a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x69a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x69e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x69e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x69e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6a200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6a200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6a200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6a600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6a600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6a600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6aa00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6aa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6aa00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6ae00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6ae00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6ae00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6b200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6b200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6b200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6b600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6b600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6b600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6ba00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6ba00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6ba00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6be00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6be00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6be00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6c200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6c200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6c200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6c600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6c600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6c600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6ca00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6ca00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6ca00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6ce00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6ce00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6ce00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6d200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6d200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6d200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6d600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6d600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6d600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6da00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6da00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6da00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6de00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6de00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6de00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6e200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6e200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6e200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6e600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6e600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6e600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6ea00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6ea00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6ea00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6ee00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6ee00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6ee00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6f200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6f200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6f200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6f600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6f600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6f600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6fa00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6fa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6fa00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6fe00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x6fe00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x6fe00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x70200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x70200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x70200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x70600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x70600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x70600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x70a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x70a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x70a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x70e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x70e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x70e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x71200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x71200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x71200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x71600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x71600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x71600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x71a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x71a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x71a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x71e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x71e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x71e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x72200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x72200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x72200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x72600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x72600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x72600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x72a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x72a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x72a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x72e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x72e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x72e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x73200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x73200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x73200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x73600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x73600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x73600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x73a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x73a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x73a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x73e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x73e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x73e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x74200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x74200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x74200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x74600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x74600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x74600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x74a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x74a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x74a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x74e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x74e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x74e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x75200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x75200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x75200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x75600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x75600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x75600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x75a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x75a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x75a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x75e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x75e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x75e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x76200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x76200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x76200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x76600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x76600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x76600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x76a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x76a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x76a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x76e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x76e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x76e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x77200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x77200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x77200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x77600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x77600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x77600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x77a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x77a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x77a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x77e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x77e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x77e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x78200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x78200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x78200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x78600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x78600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x78600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x78a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x78a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x78a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x78e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x78e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x78e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x79200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x79200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x79200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x79600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x79600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x79600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x79a00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x79a00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x79a00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x79e00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x79e00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x79e00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7a200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7a200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7a200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7a600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7a600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7a600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7aa00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7aa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7aa00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7ae00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7ae00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7ae00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7b200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7b200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7b200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7b600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7b600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7b600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7ba00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7ba00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7ba00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7be00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7be00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7be00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7c200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7c200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7c200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7c600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7c600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7c600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7ca00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7ca00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7ca00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7ce00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7ce00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7ce00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7d200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7d200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7d200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7d600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7d600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7d600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7da00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7da00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7da00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7de00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7de00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7de00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7e200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7e200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7e200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7e600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7e600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7e600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7ea00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7ea00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7ea00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7ee00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7ee00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7ee00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7f200~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7f200~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7f200~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7f600~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7f600~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7f600~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7fa00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7fa00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7fa00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7fe00~200 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x7fe00~200 size 0x52000 (335872) 2019-12-07T19:34:04.865+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 0x7fe00~200 = 0 2019-12-07T19:34:04.865+0000 7fe2c425f700 20 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=0 len=36864 2019-12-07T19:34:04.866+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:f7919c75:::smithi13140796-20:head 2019-12-07T19:34:04.866+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:f7919c75:::smithi13140796-20:head 2019-12-07T19:34:04.866+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:34:04.866+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 335872 2019-12-07T19:34:04.866+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key 2019-12-07T19:34:04.866+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key = 0 2019-12-07T19:34:04.866+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.fs0 27/26 ECSubReadReply(tid=83, attrs_read=0)) v2 -- 0x558e8402e500 con 0x558e818d7000 2019-12-07T19:34:04.866+0000 7fe2c425f700 10 osd.0 27 dequeue_op 0x558e83edcc60 finish 2019-12-07T19:34:04.866+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.866+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 174 ==== MOSDPGPush(1.8s5 27/26 [PushOp(1:131421e8:::smithi13140796-7:4, version: 27'20, data_included: [0~86016], data_size: 86016, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:4@27'20, size: 344064, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:344064, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 87034+0+0 (unknown 0 0 0) 0x558e81965200 con 0x558e818d7c00 2019-12-07T19:34:04.866+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edcf20 prio 127 cost 87016 latency 0.000029 epoch 27 MOSDPGPush(1.8s5 27/26 [PushOp(1:131421e8:::smithi13140796-7:4, version: 27'20, data_included: [0~86016], data_size: 86016, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:4@27'20, size: 344064, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:344064, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.866+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:131421e8:::smithi13140796-7:4, version: 27'20, data_included: [0~86016], data_size: 86016, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:4@27'20, size: 344064, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:344064, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 87016 e27) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:131421e8:::smithi13140796-7:4, version: 27'20, data_included: [0~86016], data_size: 86016, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:4@27'20, size: 344064, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:344064, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 87016 e27) queued 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:131421e8:::smithi13140796-7:4, version: 27'20, data_included: [0~86016], data_size: 86016, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:4@27'20, size: 344064, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:344064, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 87016 e27) pg 0x558e841a4000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e83edcf20 prio 127 cost 87016 latency 0.000077 MOSDPGPush(1.8s5 27/26 [PushOp(1:131421e8:::smithi13140796-7:4, version: 27'20, data_included: [0~86016], data_size: 86016, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:4@27'20, size: 344064, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:344064, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 27'23 lc 27'19 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=4 mbc={} ps=[2~1]] 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'19 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=4 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.8s5 27/26 [PushOp(1:131421e8:::smithi13140796-7:4, version: 27'20, data_included: [0~86016], data_size: 86016, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:4@27'20, size: 344064, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:344064, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'19 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=4 mbc={} ps=[2~1]] on_local_recover: 1:131421e8:::smithi13140796-7:4 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 snap_mapper.remove_oid 1:131421e8:::smithi13140796-7:4 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 snap_mapper._remove_oid 1:131421e8:::smithi13140796-7:4 2019-12-07T19:34:04.866+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 snap_mapper.get_snaps 1:131421e8:::smithi13140796-7:4 got.empty() 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'19 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=4 mbc={} ps=[2~1]] snapset 4=[]:{4=[4]} 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'19 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=4 mbc={} ps=[2~1]] snaps 4 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 snap_mapper.add_oid 1:131421e8:::smithi13140796-7:4 4 2019-12-07T19:34:04.866+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 snap_mapper.get_snaps 1:131421e8:::smithi13140796-7:4 got.empty() 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 snap_mapper.set_snaps 1:131421e8:::smithi13140796-7:4 4 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.8C824871.4.smithi13140796-7.. 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 snap_mapper.add_oid set SNA_1_0000000000000004_.5_0000000000000001.8C824871.4.smithi13140796-7.. 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'19 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=4 mbc={} ps=[2~1]] got missing 1:131421e8:::smithi13140796-7:4 v 27'20 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'20 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'19 active m=3 mbc={} ps=[2~1]] last_complete now 27'20 log.complete_to 27'21 2019-12-07T19:34:04.866+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e83e72c00 seq 19 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:131421e8:::smithi13140796-7:4# key 0x858000000000000001131421e821736d'ithi13140796-7!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 5#1:131421e8:::smithi13140796-7:4# 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:131421e8:::smithi13140796-7:4# key 0x858000000000000001131421e821736d'ithi13140796-7!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:131421e8:::smithi13140796-7:4# 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1141 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:131421e8:::smithi13140796-7:4# = 0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:131421e8:::smithi13140796-7:4# 0x0~15000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#1:131421e8:::smithi13140796-7:4# 0x0~15000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~15000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72c00 11 blobs 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate 0x15000/1000,15000,0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x537000~15000/1000,15000,0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x537000~15000] 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f61420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f610a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f61420 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f610a0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f61420) get_ref 0x0~2000 Blob(0x558e83f61420 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f610a0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f61420) get_ref init 0x2000, 1000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83f61420 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f610a0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f610b8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5b3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5b3b0 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b490 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5b3b0) get_ref 0x0~2000 Blob(0x558e83f5b3b0 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b490 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5b3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83f5b3b0 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5b490 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f5b4a8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5b500 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84644cb0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5b500 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644cb0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5b500) get_ref 0x0~2000 Blob(0x558e83f5b500 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644cb0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5b500) get_ref init 0x2000, 1000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e83f5b500 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644cb0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644cc8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84644d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84644d90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84644d20 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644d90 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644d20) get_ref 0x0~2000 Blob(0x558e84644d20 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644d90 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644d20) get_ref init 0x2000, 1000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e84644d20 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644d90 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644da8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84644e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84644e70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84644e00 blob([0x53f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644e70 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644e00) get_ref 0x0~2000 Blob(0x558e84644e00 blob([0x53f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644e70 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644e00) get_ref init 0x2000, 1000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e84644e00 blob([0x53f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644e70 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644e88 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84644ee0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84644f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84644ee0 blob([0x541000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644f50 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644ee0) get_ref 0x0~2000 Blob(0x558e84644ee0 blob([0x541000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644f50 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e84644ee0 blob([0x541000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644f50 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644f68 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84644fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645030 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84644fc0 blob([0x543000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645030 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644fc0) get_ref 0x0~2000 Blob(0x558e84644fc0 blob([0x543000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645030 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e84644fc0 blob([0x543000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645030 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645048 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846450a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846450a0 blob([0x545000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645110 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e846450a0) get_ref 0x0~2000 Blob(0x558e846450a0 blob([0x545000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645110 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e846450a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e846450a0 blob([0x545000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645110 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645128 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84645180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846451f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84645180 blob([0x547000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846451f0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645180) get_ref 0x0~2000 Blob(0x558e84645180 blob([0x547000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846451f0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645180) get_ref init 0x2000, 1000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e84645180 blob([0x547000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846451f0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645208 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84645260 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846452d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84645260 blob([0x549000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846452d0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645260) get_ref 0x0~2000 Blob(0x558e84645260 blob([0x549000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846452d0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645260) get_ref init 0x2000, 1000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e84645260 blob([0x549000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846452d0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e846452e8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84645340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846453b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84645340 blob([0x54b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846453b0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645340) get_ref 0x0~1000 Blob(0x558e84645340 blob([0x54b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846453b0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645340) get_ref init 0x1000, 1000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~1000: 0x0~1000 Blob(0x558e84645340 blob([0x54b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e846453b0 sbid 0x0)) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e846453c8 in 0x558e80219960) _discard 0x0~1000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x15000 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e811dc888) dirty_range mark inline shard dirty 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:131421e8:::smithi13140796-7:4# 0x0~15000 = 0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:4# 3 keys 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:4# 3 keys = 0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.8C824871.4.smithi13140796-7..' <- OBJ_.5_0000000000000001.8C824871.4.smithi13140796-7.. 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.5_0000000000000001.8C824871.4.smithi13140796-7..' <- SNA_1_0000000000000004_.5_0000000000000001.8C824871.4.smithi13140796-7.. 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x000000000000045f'.missing/0000000000000001.8C824871.4.smithi13140796-7..' <- missing/0000000000000001.8C824871.4.smithi13140796-7.. 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.866+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72c00 cost 136828 (12 ios * 4000 + 88828 bytes) 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72c00 onodes 0x558e811dc780 shared_blobs 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e811dc888) update 5#1:131421e8:::smithi13140796-7:4# 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e811dc888) update inline shard 209 bytes from 11 extents 2019-12-07T19:34:04.866+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:131421e8:::smithi13140796-7:4# is 626 (411 bytes onode + 2 bytes spanning blobs + 213 bytes inline extents) 2019-12-07T19:34:04.867+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72c00 allocated 0x[537000~15000] released 0x[] 2019-12-07T19:34:04.867+0000 7fe2c7a66700 10 freelist allocate 0x537000~15000 2019-12-07T19:34:04.867+0000 7fe2c7a66700 20 freelist _xor first_key 0x500000 last_key 0x500000 2019-12-07T19:34:04.867+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72c00 prepare 2019-12-07T19:34:04.867+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83e72c00 2019-12-07T19:34:04.867+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e83edcf20 finish 2019-12-07T19:34:04.867+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.867+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'20 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'19 active m=3 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.867+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.867+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72c00 aio_wait 2019-12-07T19:34:04.867+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72c00 2019-12-07T19:34:04.867+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72c00 io_done 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.867+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2373 < min_flush_size 524288 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dc780 had 1 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.867+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2396 < min_flush_size 524288 2019-12-07T19:34:04.867+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2396 < min_flush_size 524288 2019-12-07T19:34:04.867+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.867+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xd34f4~95c to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.867+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xd34f4 0xd34f4~95c to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa34f4 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x4f4 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xe50 and padding block with 0x1b0 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xd3e50 2019-12-07T19:34:04.867+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.867+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.867+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000201859s (0.000090019s flush + 0.000111840s kv commit) 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.867+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e72c00> 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.867+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72c00 kv_submitted 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72c00 2019-12-07T19:34:04.867+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72c00 finishing 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72c00 onodes 0x558e811dc780 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5b4a8 in 0x558e80219960) _finish_write discard buffer(0x558e8456efc0 space 0x558e83f5b4a8 0x0~2000 writing nocache) 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f610b8 in 0x558e80219960) _finish_write discard buffer(0x558e8456ef30 space 0x558e83f610b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.867+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84644cc8 in 0x558e80219960) _finish_write discard buffer(0x558e8456f050 space 0x558e84644cc8 0x0~2000 writing nocache) 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84644da8 in 0x558e80219960) _finish_write discard buffer(0x558e8456f0e0 space 0x558e84644da8 0x0~2000 writing nocache) 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84644e88 in 0x558e80219960) _finish_write discard buffer(0x558e8456f170 space 0x558e84644e88 0x0~2000 writing nocache) 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84644f68 in 0x558e80219960) _finish_write discard buffer(0x558e8456f200 space 0x558e84644f68 0x0~2000 writing nocache) 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645048 in 0x558e80219960) _finish_write discard buffer(0x558e8456f290 space 0x558e84645048 0x0~2000 writing nocache) 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645128 in 0x558e80219960) _finish_write discard buffer(0x558e8456f320 space 0x558e84645128 0x0~2000 writing nocache) 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645208 in 0x558e80219960) _finish_write discard buffer(0x558e8456f3b0 space 0x558e84645208 0x0~2000 writing nocache) 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846452e8 in 0x558e80219960) _finish_write discard buffer(0x558e842b8ab0 space 0x558e846452e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846453c8 in 0x558e80219960) _finish_write discard buffer(0x558e842b8b40 space 0x558e846453c8 0x0~1000 writing nocache) 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72c00 done 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.867+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72c00 [] 2019-12-07T19:34:04.867+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.867+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.867+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'20 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'19 active m=3 mbc={} ps=[2~1]] recovery_committed_to version 27'20 now ondisk 2019-12-07T19:34:04.867+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 27/26 [PushReplyOp(1:131421e8:::smithi13140796-7:4)]) v3 -- 0x558e818ee600 con 0x558e818d7c00 2019-12-07T19:34:04.867+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.869+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 100 ==== MRecoveryReserve(1.fs1 RELEASE e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e84199860 con 0x558e818d7000 2019-12-07T19:34:04.869+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.fs1 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.869+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.869+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.869+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.869+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.869+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558e81a9b000 2019-12-07T19:34:04.869+0000 7fe2c8267700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.869+0000 7fe2c8267700 5 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.870607+0000 1 0.000039 2019-12-07T19:34:04.869+0000 7fe2c8267700 5 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.869+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:04.869+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:04.869+0000 7fe2cea74700 15 osd.0 27 enqueue_peering_evt 1.as3 epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.869+0000 7fe2cea74700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) 2019-12-07T19:34:04.869+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.869+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.869+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.869+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved) prio 255 cost 10 e27) pg 0x558e8193b000 2019-12-07T19:34:04.869+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RemoteRecoveryReserved 2019-12-07T19:34:04.869+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MRecoveryReserve(1.as0 GRANT e27/27) v3 -- 0x558e819b0780 con 0x558e818d7c00 2019-12-07T19:34:04.870+0000 7fe2caa6c700 5 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepWaitRecoveryReserved 2019-12-07T19:34:04.870699+0000 1 0.000028 2019-12-07T19:34:04.870+0000 7fe2caa6c700 5 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.870+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.870+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.872+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 175 ==== MOSDECSubOpRead(1.as3 27/26 ECSubRead(tid=130, to_read={1:51ccf92f:::smithi13140796-15:head=0,262144,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8402e500 con 0x558e818d7c00 2019-12-07T19:34:04.872+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edcdc0 prio 3 cost 0 latency 0.000022 epoch 27 MOSDECSubOpRead(1.as3 27/26 ECSubRead(tid=130, to_read={1:51ccf92f:::smithi13140796-15:head=0,262144,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.872+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpRead(1.as3 27/26 ECSubRead(tid=130, to_read={1:51ccf92f:::smithi13140796-15:head=0,262144,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpRead(1.as3 27/26 ECSubRead(tid=130, to_read={1:51ccf92f:::smithi13140796-15:head=0,262144,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) queued 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGOpItem(op=MOSDECSubOpRead(1.as3 27/26 ECSubRead(tid=130, to_read={1:51ccf92f:::smithi13140796-15:head=0,262144,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3) prio 3 cost 0 e27) pg 0x558e8193b000 2019-12-07T19:34:04.872+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edcdc0 prio 3 cost 0 latency 0.000079 MOSDECSubOpRead(1.as3 27/26 ECSubRead(tid=130, to_read={1:51ccf92f:::smithi13140796-15:head=0,262144,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] 2019-12-07T19:34:04.872+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] _handle_message: MOSDECSubOpRead(1.as3 27/26 ECSubRead(tid=130, to_read={1:51ccf92f:::smithi13140796-15:head=0,262144,0}, subchunks={1:51ccf92f:::smithi13140796-15:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:04.872+0000 7fe2c6a64700 25 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:04.872+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# 0x0~40000 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~40000 size 0xf000 (61440) 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e92d0 blob([0x6a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64f50 sbid 0x0)) need 0x0~2000 cache has 0x[0~1000] 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9880 blob([0x6c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e98f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9960 blob([0x6e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9a40 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9ab0 blob([0x70000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65f80 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac4000 blob([0x72000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4070 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac40e0 blob([0x74000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4150 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac41c0 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4230 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83ac42a0 blob([0x78000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83ac4310 sbid 0x0)) need 0x0~1000 cache has 0x[] 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e92d0 blob([0x6a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64f50 sbid 0x0)) need {<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1000: 0x1000 reading 0x1000~1000 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e9880 blob([0x6c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e98f0 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e9960 blob([0x6e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9a40 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e817e9ab0 blob([0x70000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65f80 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac4000 blob([0x72000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4070 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac40e0 blob([0x74000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4150 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac41c0 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4230 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e83ac42a0 blob([0x78000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83ac4310 sbid 0x0)) need {<0x0, 0x1000> : [0xe000:0~1000]} 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~1000 2019-12-07T19:34:04.872+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e92d0 blob([0x6a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64f50 sbid 0x0)) need 0x{<0x1000, 0x1000> : [0x1000:1000~1000]} 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83c64f68 in 0x558e80219b20) _discard 0x1000~1000 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 _add level 1 near 0 on buffer(0x558e84032510 space 0x558e83c64f68 0x1000~1000 clean) which has cache_private 0 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e9880 blob([0x6c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e98f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e817e9908 in 0x558e80219b20) _discard 0x0~2000 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 _add level 1 near 0 on buffer(0x558e83d10c60 space 0x558e817e9908 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e9960 blob([0x6e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9a40 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e817e9a58 in 0x558e80219b20) _discard 0x0~2000 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 _add level 1 near 0 on buffer(0x558e83d10d80 space 0x558e817e9a58 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e817e9ab0 blob([0x70000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65f80 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83c65f98 in 0x558e80219b20) _discard 0x0~2000 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 _add level 1 near 0 on buffer(0x558e83e75a70 space 0x558e83c65f98 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac4000 blob([0x72000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4070 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83ac4088 in 0x558e80219b20) _discard 0x0~2000 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 _add level 1 near 0 on buffer(0x558e83e75d40 space 0x558e83ac4088 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac40e0 blob([0x74000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4150 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83ac4168 in 0x558e80219b20) _discard 0x0~2000 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 _add level 1 near 0 on buffer(0x558e83e75dd0 space 0x558e83ac4168 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac41c0 blob([0x76000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4230 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83ac4248 in 0x558e80219b20) _discard 0x0~2000 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 _add level 1 near 0 on buffer(0x558e83e75e60 space 0x558e83ac4248 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e83ac42a0 blob([0x78000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83ac4310 sbid 0x0)) need 0x{<0x0, 0x1000> : [0xe000:0~1000]} 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 bluestore.BufferSpace(0x558e83ac4328 in 0x558e80219b20) _discard 0x0~1000 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 _add level 1 near 0 on buffer(0x558e83e75ef0 space 0x558e83ac4328 0x0~1000 clean) which has cache_private 0 2019-12-07T19:34:04.873+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# 0x0~40000 = 61440 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] handle_sub_read read request=262144 r=61440 len=61440 2019-12-07T19:34:04.873+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] get_hash_info: Getting attr on 1:51ccf92f:::smithi13140796-15:head 2019-12-07T19:34:04.873+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] get_hash_info: not in cache 1:51ccf92f:::smithi13140796-15:head 2019-12-07T19:34:04.873+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# 2019-12-07T19:34:04.873+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] get_hash_info: found on disk, size 61440 2019-12-07T19:34:04.873+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# hinfo_key 2019-12-07T19:34:04.873+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.as3_head 3#1:51ccf92f:::smithi13140796-15:head# hinfo_key = 0 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] handle_sub_read: Checking hash of 1:51ccf92f:::smithi13140796-15:head 2019-12-07T19:34:04.873+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.as0 27/26 ECSubReadReply(tid=130, attrs_read=0)) v2 -- 0x558e84242a00 con 0x558e818d7c00 2019-12-07T19:34:04.873+0000 7fe2c6a64700 10 osd.0 27 dequeue_op 0x558e83edcdc0 finish 2019-12-07T19:34:04.873+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.876+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 176 ==== MOSDPGPush(1.8s5 27/26 [PushOp(1:131421e8:::smithi13140796-7:head, version: 27'21, data_included: [0~126976], data_size: 126976, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:head@27'21, size: 507904, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:507904, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 128090+0+0 (unknown 0 0 0) 0x558e818ee600 con 0x558e818d7c00 2019-12-07T19:34:04.876+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edc9a0 prio 127 cost 127976 latency 0.000033 epoch 27 MOSDPGPush(1.8s5 27/26 [PushOp(1:131421e8:::smithi13140796-7:head, version: 27'21, data_included: [0~126976], data_size: 126976, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:head@27'21, size: 507904, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:507904, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.876+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:131421e8:::smithi13140796-7:head, version: 27'21, data_included: [0~126976], data_size: 126976, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:head@27'21, size: 507904, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:507904, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 127976 e27) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:131421e8:::smithi13140796-7:head, version: 27'21, data_included: [0~126976], data_size: 126976, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:head@27'21, size: 507904, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:507904, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 127976 e27) queued 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:131421e8:::smithi13140796-7:head, version: 27'21, data_included: [0~126976], data_size: 126976, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:head@27'21, size: 507904, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:507904, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 127976 e27) pg 0x558e841a4000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e83edc9a0 prio 127 cost 127976 latency 0.000092 MOSDPGPush(1.8s5 27/26 [PushOp(1:131421e8:::smithi13140796-7:head, version: 27'21, data_included: [0~126976], data_size: 126976, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:head@27'21, size: 507904, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:507904, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 27'23 lc 27'20 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=3 mbc={} ps=[2~1]] 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'20 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=3 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.8s5 27/26 [PushOp(1:131421e8:::smithi13140796-7:head, version: 27'21, data_included: [0~126976], data_size: 126976, omap_header_size: 0, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(1:131421e8:::smithi13140796-7:head@27'21, size: 507904, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:507904, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'20 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=3 mbc={} ps=[2~1]] on_local_recover: 1:131421e8:::smithi13140796-7:head 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'20 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=3 mbc={} ps=[2~1]] got missing 1:131421e8:::smithi13140796-7:head v 27'21 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'21 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'20 active m=2 mbc={} ps=[2~1]] last_complete now 27'21 log.complete_to 27'22 2019-12-07T19:34:04.876+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e8417a900 seq 20 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:131421e8:::smithi13140796-7:head# key 0x858000000000000001131421e821736d'ithi13140796-7!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:131421e8:::smithi13140796-7:head# key 0x858000000000000001131421e821736d'ithi13140796-7!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.876+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1142 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# = 0 2019-12-07T19:34:04.876+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 0x0~1f000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#1:131421e8:::smithi13140796-7:head# 0x0~1f000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1f000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8417a900 16 blobs 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate 0x1f000/1000,1f000,0 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x54c000~1f000/1000,1f000,0 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x54c000~1f000] 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f61570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f615e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f61570 blob([0x54c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f615e0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f61570) get_ref 0x0~2000 Blob(0x558e83f61570 blob([0x54c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f615e0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f61570) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83f61570 blob([0x54c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f615e0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f615f8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f613b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f61260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f613b0 blob([0x54e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61260 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f613b0) get_ref 0x0~2000 Blob(0x558e83f613b0 blob([0x54e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61260 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f613b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83f613b0 blob([0x54e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f61260 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f61278 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846458f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846458f0 blob([0x550000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645960 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e846458f0) get_ref 0x0~2000 Blob(0x558e846458f0 blob([0x550000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645960 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e846458f0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e846458f0 blob([0x550000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645960 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645978 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846459d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846459d0 blob([0x552000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645a40 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e846459d0) get_ref 0x0~2000 Blob(0x558e846459d0 blob([0x552000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645a40 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e846459d0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e846459d0 blob([0x552000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645a40 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645a58 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84645ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84645ab0 blob([0x554000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645b20 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645ab0) get_ref 0x0~2000 Blob(0x558e84645ab0 blob([0x554000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645b20 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e84645ab0 blob([0x554000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645b20 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645b38 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84645b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84645b90 blob([0x556000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645c00 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645b90) get_ref 0x0~2000 Blob(0x558e84645b90 blob([0x556000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645c00 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645b90) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e84645b90 blob([0x556000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645c00 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645c18 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84645c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645ce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84645c70 blob([0x558000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645ce0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645c70) get_ref 0x0~2000 Blob(0x558e84645c70 blob([0x558000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645ce0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645c70) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e84645c70 blob([0x558000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645ce0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645cf8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84645d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645dc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84645d50 blob([0x55a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645dc0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645d50) get_ref 0x0~2000 Blob(0x558e84645d50 blob([0x55a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645dc0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645d50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e84645d50 blob([0x55a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645dc0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645dd8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84645e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84645e30 blob([0x55c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645ea0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645e30) get_ref 0x0~2000 Blob(0x558e84645e30 blob([0x55c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645ea0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645e30) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e84645e30 blob([0x55c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645ea0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645eb8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84645f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84645f10 blob([0x55e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645f80 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645f10) get_ref 0x0~2000 Blob(0x558e84645f10 blob([0x55e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645f80 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645f10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e84645f10 blob([0x55e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645f80 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645f98 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f60f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f61340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f60f50 blob([0x560000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61340 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f60f50) get_ref 0x0~2000 Blob(0x558e83f60f50 blob([0x560000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61340 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f60f50) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83f60f50 blob([0x560000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f61340 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f61358 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f94bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b284d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f94bd0 blob([0x562000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b284d0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f94bd0) get_ref 0x0~2000 Blob(0x558e83f94bd0 blob([0x562000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b284d0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f94bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83f94bd0 blob([0x562000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b284d0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83b284e8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 177 ==== MRecoveryReserve(1.as3 RELEASE e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e819b0780 con 0x558e818d7c00 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84644c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84644bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84644c40 blob([0x564000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644bd0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.as3 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644c40) get_ref 0x0~2000 Blob(0x558e84644c40 blob([0x564000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644bd0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644c40) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e84644c40 blob([0x564000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644bd0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644be8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84644b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84644af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84644b60 blob([0x566000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644af0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644b60) get_ref 0x0~2000 Blob(0x558e84644b60 blob([0x566000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644af0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644b60) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e84644b60 blob([0x566000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644af0 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644b08 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.876+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.876+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558e8193b000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84644a80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84644a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84644a80 blob([0x568000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644a10 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644a80) get_ref 0x0~2000 Blob(0x558e84644a80 blob([0x568000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644a10 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644a80) get_ref init 0x2000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e84644a80 blob([0x568000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644a10 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644a28 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846449a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84644930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.876+0000 7fe2caa6c700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846449a0 blob([0x56a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644930 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e846449a0) get_ref 0x0~1000 Blob(0x558e846449a0 blob([0x56a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644930 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.blob(0x558e846449a0) get_ref init 0x1000, 1000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~1000: 0x0~1000 Blob(0x558e846449a0 blob([0x56a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e84644930 sbid 0x0)) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644948 in 0x558e80219960) _discard 0x0~1000 2019-12-07T19:34:04.876+0000 7fe2caa6c700 5 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.877555+0000 1 0.000040 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1f000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e83b09508) dirty_range mark inline shard dirty 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 0x0~1f000 = 0 2019-12-07T19:34:04.876+0000 7fe2caa6c700 5 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.876+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 4 keys 2019-12-07T19:34:04.876+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:04.876+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 4 keys = 0 2019-12-07T19:34:04.876+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x000000000000045f'.missing/0000000000000001.8C824871.head.smithi13140796-7..' <- missing/0000000000000001.8C824871.head.smithi13140796-7.. 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.876+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417a900 cost 197384 (17 ios * 4000 + 129384 bytes) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417a900 onodes 0x558e83b09400 shared_blobs 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e83b09508) update 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e83b09508) update inline shard 304 bytes from 16 extents 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:131421e8:::smithi13140796-7:head# is 850 (540 bytes onode + 2 bytes spanning blobs + 308 bytes inline extents) 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417a900 allocated 0x[54c000~1f000] released 0x[] 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 freelist allocate 0x54c000~1f000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 20 freelist _xor first_key 0x500000 last_key 0x500000 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a900 prepare 2019-12-07T19:34:04.876+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8417a900 2019-12-07T19:34:04.877+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e83edc9a0 finish 2019-12-07T19:34:04.877+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.877+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'21 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'20 active m=2 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.877+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.877+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a900 aio_wait 2019-12-07T19:34:04.877+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417a900 2019-12-07T19:34:04.877+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a900 io_done 2019-12-07T19:34:04.877+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.877+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.877+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.877+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2289 < min_flush_size 524288 2019-12-07T19:34:04.877+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b09400 had 1 2019-12-07T19:34:04.877+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.877+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2312 < min_flush_size 524288 2019-12-07T19:34:04.877+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2312 < min_flush_size 524288 2019-12-07T19:34:04.877+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.877+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xd3e50~908 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.877+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xd3e50 0xd3e50~908 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.877+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.877+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa3e50 2019-12-07T19:34:04.877+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xe50 2019-12-07T19:34:04.877+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.877+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x758 and padding block with 0x8a8 2019-12-07T19:34:04.877+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xd4758 2019-12-07T19:34:04.877+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.877+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.877+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.877+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.877+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000143036s (0.000041464s flush + 0.000101572s kv commit) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.877+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417a900> 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.877+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a900 kv_submitted 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417a900 2019-12-07T19:34:04.877+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a900 finishing 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417a900 onodes 0x558e83b09400 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b284e8 in 0x558e80219960) _finish_write discard buffer(0x558e842b9320 space 0x558e83b284e8 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f61278 in 0x558e80219960) _finish_write discard buffer(0x558e842b8d80 space 0x558e83f61278 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f61358 in 0x558e80219960) _finish_write discard buffer(0x558e842b9290 space 0x558e83f61358 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f615f8 in 0x558e80219960) _finish_write discard buffer(0x558e842b8cf0 space 0x558e83f615f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84644948 in 0x558e80219960) _finish_write discard buffer(0x558e842b9560 space 0x558e84644948 0x0~1000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84644a28 in 0x558e80219960) _finish_write discard buffer(0x558e842b94d0 space 0x558e84644a28 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84644b08 in 0x558e80219960) _finish_write discard buffer(0x558e842b9440 space 0x558e84644b08 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84644be8 in 0x558e80219960) _finish_write discard buffer(0x558e842b93b0 space 0x558e84644be8 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645978 in 0x558e80219960) _finish_write discard buffer(0x558e842b8e10 space 0x558e84645978 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645a58 in 0x558e80219960) _finish_write discard buffer(0x558e842b8ea0 space 0x558e84645a58 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'21 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'20 active m=2 mbc={} ps=[2~1]] recovery_committed_to version 27'21 now ondisk 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645b38 in 0x558e80219960) _finish_write discard buffer(0x558e842b8f30 space 0x558e84645b38 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645c18 in 0x558e80219960) _finish_write discard buffer(0x558e842b8fc0 space 0x558e84645c18 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645cf8 in 0x558e80219960) _finish_write discard buffer(0x558e842b9050 space 0x558e84645cf8 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645dd8 in 0x558e80219960) _finish_write discard buffer(0x558e842b90e0 space 0x558e84645dd8 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645eb8 in 0x558e80219960) _finish_write discard buffer(0x558e842b9170 space 0x558e84645eb8 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645f98 in 0x558e80219960) _finish_write discard buffer(0x558e842b9200 space 0x558e84645f98 0x0~2000 writing nocache) 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a900 done 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.877+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 27/26 [PushReplyOp(1:131421e8:::smithi13140796-7:head)]) v3 -- 0x558e81965200 con 0x558e818d7c00 2019-12-07T19:34:04.877+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417a900 [] 2019-12-07T19:34:04.877+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.877+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.877+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.882+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 178 ==== MOSDPGPush(1.8s5 27/26 [PushOp(1:15219530:::smithi13140796-21:4, version: 27'22, data_included: [0~86016], data_size: 86016, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:4@27'22, size: 344064, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:344064, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 87038+0+0 (unknown 0 0 0) 0x558e81965200 con 0x558e818d7c00 2019-12-07T19:34:04.882+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edd080 prio 127 cost 87016 latency 0.000037 epoch 27 MOSDPGPush(1.8s5 27/26 [PushOp(1:15219530:::smithi13140796-21:4, version: 27'22, data_included: [0~86016], data_size: 86016, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:4@27'22, size: 344064, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:344064, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.882+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:15219530:::smithi13140796-21:4, version: 27'22, data_included: [0~86016], data_size: 86016, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:4@27'22, size: 344064, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:344064, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 87016 e27) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:15219530:::smithi13140796-21:4, version: 27'22, data_included: [0~86016], data_size: 86016, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:4@27'22, size: 344064, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:344064, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 87016 e27) queued 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:15219530:::smithi13140796-21:4, version: 27'22, data_included: [0~86016], data_size: 86016, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:4@27'22, size: 344064, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:344064, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 87016 e27) pg 0x558e841a4000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e83edd080 prio 127 cost 87016 latency 0.000085 MOSDPGPush(1.8s5 27/26 [PushOp(1:15219530:::smithi13140796-21:4, version: 27'22, data_included: [0~86016], data_size: 86016, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:4@27'22, size: 344064, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:344064, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 27'23 lc 27'21 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=2 mbc={} ps=[2~1]] 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'21 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=2 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.8s5 27/26 [PushOp(1:15219530:::smithi13140796-21:4, version: 27'22, data_included: [0~86016], data_size: 86016, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:4@27'22, size: 344064, copy_subset: [], clone_subset: {}, snapset: 4=[]:{4=[4]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:344064, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'21 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=2 mbc={} ps=[2~1]] on_local_recover: 1:15219530:::smithi13140796-21:4 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 snap_mapper.remove_oid 1:15219530:::smithi13140796-21:4 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 snap_mapper._remove_oid 1:15219530:::smithi13140796-21:4 2019-12-07T19:34:04.882+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 snap_mapper.get_snaps 1:15219530:::smithi13140796-21:4 got.empty() 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'21 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=2 mbc={} ps=[2~1]] snapset 4=[]:{4=[4]} 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'21 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=2 mbc={} ps=[2~1]] snaps 4 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 snap_mapper.add_oid 1:15219530:::smithi13140796-21:4 4 2019-12-07T19:34:04.882+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 snap_mapper.get_snaps 1:15219530:::smithi13140796-21:4 got.empty() 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 snap_mapper.set_snaps 1:15219530:::smithi13140796-21:4 4 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.8A489AC0.4.smithi13140796-21.. 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 snap_mapper.add_oid set SNA_1_0000000000000004_.5_0000000000000001.8A489AC0.4.smithi13140796-21.. 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'21 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=2 mbc={} ps=[2~1]] got missing 1:15219530:::smithi13140796-21:4 v 27'22 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'22 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'21 active m=1 mbc={} ps=[2~1]] last_complete now 27'22 log.complete_to 27'23 2019-12-07T19:34:04.882+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e83c56f00 seq 21 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:15219530:::smithi13140796-21:4# key 0x858000000000000001152195'0!smithi13140796-21!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 5#1:15219530:::smithi13140796-21:4# 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:15219530:::smithi13140796-21:4# key 0x858000000000000001152195'0!smithi13140796-21!='0x0000000000000004ffffffffffffffff'o' 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:15219530:::smithi13140796-21:4# 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1143 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:15219530:::smithi13140796-21:4# = 0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:15219530:::smithi13140796-21:4# 0x0~15000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#1:15219530:::smithi13140796-21:4# 0x0~15000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~15000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83c56f00 11 blobs 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate 0x15000/1000,15000,0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x56b000~15000/1000,15000,0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x56b000~15000] 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f617a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f617a0 blob([0x56b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b420 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f617a0) get_ref 0x0~2000 Blob(0x558e83f617a0 blob([0x56b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b420 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f617a0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e83f617a0 blob([0x56b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5b420 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f5b438 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f61490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84548540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f61490 blob([0x56d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84548540 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f61490) get_ref 0x0~2000 Blob(0x558e83f61490 blob([0x56d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84548540 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f61490) get_ref init 0x2000, 1000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e83f61490 blob([0x56d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84548540 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548558 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e845485b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84548620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e845485b0 blob([0x56f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84548620 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e845485b0) get_ref 0x0~2000 Blob(0x558e845485b0 blob([0x56f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84548620 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e845485b0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e845485b0 blob([0x56f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84548620 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548638 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84548690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84548700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84548690 blob([0x571000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84548700 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548690) get_ref 0x0~2000 Blob(0x558e84548690 blob([0x571000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84548700 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548690) get_ref init 0x2000, 1000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e84548690 blob([0x571000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84548700 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548718 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84548770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e845487e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84548770 blob([0x573000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e845487e0 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548770) get_ref 0x0~2000 Blob(0x558e84548770 blob([0x573000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e845487e0 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548770) get_ref init 0x2000, 1000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e84548770 blob([0x573000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e845487e0 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e845487f8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84548850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e845488c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84548850 blob([0x575000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e845488c0 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548850) get_ref 0x0~2000 Blob(0x558e84548850 blob([0x575000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e845488c0 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548850) get_ref init 0x2000, 1000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e84548850 blob([0x575000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e845488c0 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e845488d8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84548930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e845489a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84548930 blob([0x577000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e845489a0 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548930) get_ref 0x0~2000 Blob(0x558e84548930 blob([0x577000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e845489a0 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548930) get_ref init 0x2000, 1000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e84548930 blob([0x577000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e845489a0 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e845489b8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84548a10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84548a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84548a10 blob([0x579000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84548a80 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548a10) get_ref 0x0~2000 Blob(0x558e84548a10 blob([0x579000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84548a80 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548a10) get_ref init 0x2000, 1000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e84548a10 blob([0x579000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84548a80 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548a98 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84548af0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84548b60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84548af0 blob([0x57b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84548b60 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548af0) get_ref 0x0~2000 Blob(0x558e84548af0 blob([0x57b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84548b60 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548af0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e84548af0 blob([0x57b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84548b60 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548b78 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84548bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84548c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84548bd0 blob([0x57d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84548c40 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548bd0) get_ref 0x0~2000 Blob(0x558e84548bd0 blob([0x57d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84548c40 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e84548bd0 blob([0x57d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84548c40 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548c58 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84548cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84548d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84548cb0 blob([0x57f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84548d20 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548cb0) get_ref 0x0~1000 Blob(0x558e84548cb0 blob([0x57f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84548d20 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548cb0) get_ref init 0x1000, 1000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~1000: 0x0~1000 Blob(0x558e84548cb0 blob([0x57f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e84548d20 sbid 0x0)) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548d38 in 0x558e80219960) _discard 0x0~1000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x15000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e84315a08) dirty_range mark inline shard dirty 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:15219530:::smithi13140796-21:4# 0x0~15000 = 0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:15219530:::smithi13140796-21:4# 3 keys 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:15219530:::smithi13140796-21:4# 3 keys = 0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.8A489AC0.4.smithi13140796-21..' <- OBJ_.5_0000000000000001.8A489AC0.4.smithi13140796-21.. 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.5_0000000000000001.8A489AC0.4.smithi13140796-21..' <- SNA_1_0000000000000004_.5_0000000000000001.8A489AC0.4.smithi13140796-21.. 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x000000000000045f'.missing/0000000000000001.8A489AC0.4.smithi13140796-21..' <- missing/0000000000000001.8A489AC0.4.smithi13140796-21.. 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56f00 cost 136835 (12 ios * 4000 + 88835 bytes) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56f00 onodes 0x558e84315900 shared_blobs 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e84315a08) update 5#1:15219530:::smithi13140796-21:4# 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e84315a08) update inline shard 209 bytes from 11 extents 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:15219530:::smithi13140796-21:4# is 627 (412 bytes onode + 2 bytes spanning blobs + 213 bytes inline extents) 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56f00 allocated 0x[56b000~15000] released 0x[] 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 freelist allocate 0x56b000~15000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 freelist _xor first_key 0x500000 last_key 0x500000 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 prepare 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83c56f00 2019-12-07T19:34:04.882+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e83edd080 finish 2019-12-07T19:34:04.882+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.882+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'22 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'21 active m=1 mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.883+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.883+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 aio_wait 2019-12-07T19:34:04.883+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56f00 2019-12-07T19:34:04.883+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 io_done 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:04.883+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2380 < min_flush_size 524288 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84315900 had 1 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.883+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2403 < min_flush_size 524288 2019-12-07T19:34:04.883+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2403 < min_flush_size 524288 2019-12-07T19:34:04.883+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.883+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xd4758~963 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.883+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xd4758 0xd4758~963 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa4758 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x758 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xbb and padding block with 0xf45 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xd50bb 2019-12-07T19:34:04.883+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.883+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.883+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000158449s (0.000047570s flush + 0.000110879s kv commit) 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56f00> 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.883+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 kv_submitted 2019-12-07T19:34:04.883+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56f00 2019-12-07T19:34:04.883+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 finishing 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56f00 onodes 0x558e84315900 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5b438 in 0x558e80219960) _finish_write discard buffer(0x558e842b8c60 space 0x558e83f5b438 0x0~2000 writing nocache) 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84548558 in 0x558e80219960) _finish_write discard buffer(0x558e842b9680 space 0x558e84548558 0x0~2000 writing nocache) 2019-12-07T19:34:04.883+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84548638 in 0x558e80219960) _finish_write discard buffer(0x558e842b9710 space 0x558e84548638 0x0~2000 writing nocache) 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84548718 in 0x558e80219960) _finish_write discard buffer(0x558e842b97a0 space 0x558e84548718 0x0~2000 writing nocache) 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e845487f8 in 0x558e80219960) _finish_write discard buffer(0x558e842b9830 space 0x558e845487f8 0x0~2000 writing nocache) 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e845488d8 in 0x558e80219960) _finish_write discard buffer(0x558e842b98c0 space 0x558e845488d8 0x0~2000 writing nocache) 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e845489b8 in 0x558e80219960) _finish_write discard buffer(0x558e842b9950 space 0x558e845489b8 0x0~2000 writing nocache) 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84548a98 in 0x558e80219960) _finish_write discard buffer(0x558e842b99e0 space 0x558e84548a98 0x0~2000 writing nocache) 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84548b78 in 0x558e80219960) _finish_write discard buffer(0x558e842b9a70 space 0x558e84548b78 0x0~2000 writing nocache) 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84548c58 in 0x558e80219960) _finish_write discard buffer(0x558e842b9b00 space 0x558e84548c58 0x0~2000 writing nocache) 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84548d38 in 0x558e80219960) _finish_write discard buffer(0x558e842b9b90 space 0x558e84548d38 0x0~1000 writing nocache) 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56f00 done 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.883+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56f00 [] 2019-12-07T19:34:04.883+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.883+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'22 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'21 active m=1 mbc={} ps=[2~1]] recovery_committed_to version 27'22 now ondisk 2019-12-07T19:34:04.883+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.883+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 27/26 [PushReplyOp(1:15219530:::smithi13140796-21:4)]) v3 -- 0x558e818ee600 con 0x558e818d7c00 2019-12-07T19:34:04.883+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.886+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 179 ==== MOSDPGPush(1.8s5 27/26 [PushOp(1:15219530:::smithi13140796-21:head, version: 27'23, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:head@27'23, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 ==== 1030+0+0 (unknown 0 0 0) 0x558e818ee600 con 0x558e818d7c00 2019-12-07T19:34:04.886+0000 7fe2e7d3c700 15 osd.0 27 enqueue_op 0x558e83edc2c0 prio 127 cost 1000 latency 0.000025 epoch 27 MOSDPGPush(1.8s5 27/26 [PushOp(1:15219530:::smithi13140796-21:head, version: 27'23, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:head@27'23, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.886+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:15219530:::smithi13140796-21:head, version: 27'23, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:head@27'23, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 1000 e27) 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:15219530:::smithi13140796-21:head, version: 27'23, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:head@27'23, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 1000 e27) queued 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDPGPush(1.8s5 27/26 [PushOp(1:15219530:::smithi13140796-21:head, version: 27'23, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:head@27'23, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4) prio 127 cost 1000 e27) pg 0x558e841a4000 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e83edc2c0 prio 127 cost 1000 latency 0.000063 MOSDPGPush(1.8s5 27/26 [PushOp(1:15219530:::smithi13140796-21:head, version: 27'23, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:head@27'23, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 pg pg[1.8s5( v 27'23 lc 27'22 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=1 mbc={} ps=[2~1]] 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'22 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=1 mbc={} ps=[2~1]] _handle_message: MOSDPGPush(1.8s5 27/26 [PushOp(1:15219530:::smithi13140796-21:head, version: 27'23, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(1:15219530:::smithi13140796-21:head@27'23, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))]) v4 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'22 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=1 mbc={} ps=[2~1]] on_local_recover: 1:15219530:::smithi13140796-21:head 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 lc 27'22 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active m=1 mbc={} ps=[2~1]] got missing 1:15219530:::smithi13140796-21:head v 27'23 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'22 active mbc={} ps=[2~1]] last_complete now 27'23 log.complete_to at end 2019-12-07T19:34:04.886+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e83c56600 seq 22 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:15219530:::smithi13140796-21:head# key 0x858000000000000001152195'0!smithi13140796-21!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_add_transaction op 13 got ENOENT on 5#1:15219530:::smithi13140796-21:head# 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:15219530:::smithi13140796-21:head# key 0x858000000000000001152195'0!smithi13140796-21!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:04.886+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1144 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:15219530:::smithi13140796-21:head# = 0 2019-12-07T19:34:04.886+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 3 keys 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 3 keys = 0 2019-12-07T19:34:04.886+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x000000000000045f'.missing/0000000000000001.8A489AC0.head.smithi13140796-21..' <- missing/0000000000000001.8A489AC0.head.smithi13140796-21.. 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.886+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56600 cost 6263 (1 ios * 4000 + 2263 bytes) 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56600 onodes 0x558e84315b80 shared_blobs 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e84315c88) update 5#1:15219530:::smithi13140796-21:head# 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e84315c88) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:15219530:::smithi13140796-21:head# is 475 (467 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56600 allocated 0x[] released 0x[] 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 prepare 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56600 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 io_done 2019-12-07T19:34:04.886+0000 7fe2c7a66700 10 osd.0 27 dequeue_op 0x558e83edc2c0 finish 2019-12-07T19:34:04.886+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:04.886+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.886+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1834 < min_flush_size 524288 2019-12-07T19:34:04.886+0000 7fe2cba6e700 20 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'22 active mbc={} ps=[2~1]] _applied_recovered_object_replica 2019-12-07T19:34:04.886+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84315b80 had 1 2019-12-07T19:34:04.886+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:04.886+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.886+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1857 < min_flush_size 524288 2019-12-07T19:34:04.886+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1857 < min_flush_size 524288 2019-12-07T19:34:04.886+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.886+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xd50bb~741 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.886+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xd50bb 0xd50bb~741 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.886+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:04.886+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa50bb 2019-12-07T19:34:04.886+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xbb 2019-12-07T19:34:04.886+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:04.886+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x7fc and padding block with 0x804 2019-12-07T19:34:04.886+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xd57fc 2019-12-07T19:34:04.886+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:04.886+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:04.886+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:04.886+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:04.886+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000117658s (0.000000114s flush + 0.000117544s kv commit) 2019-12-07T19:34:04.886+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:04.886+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:04.886+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56600> 2019-12-07T19:34:04.886+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:04.886+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 kv_submitted 2019-12-07T19:34:04.886+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56600 2019-12-07T19:34:04.886+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 finishing 2019-12-07T19:34:04.886+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56600 onodes 0x558e84315b80 2019-12-07T19:34:04.886+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56600 done 2019-12-07T19:34:04.886+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:04.886+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56600 [] 2019-12-07T19:34:04.886+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.886+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:04.886+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:04.886+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 lcod 27'22 active mbc={} ps=[2~1]] recovery_committed_to version 27'23 now ondisk 2019-12-07T19:34:04.886+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_trim(1.8s0 27'23 e27/27) v2 -- 0x558e8403fa40 con 0x558e818d7c00 2019-12-07T19:34:04.886+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDPGPushReply(1.8s0 27/26 [PushReplyOp(1:15219530:::smithi13140796-21:head)]) v3 -- 0x558e81965200 con 0x558e818d7c00 2019-12-07T19:34:04.886+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.887+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 180 ==== MRecoveryReserve(1.8s5 RELEASE e27/27) v3 ==== 30+0+0 (unknown 0 0 0) 0x558e84199860 con 0x558e818d7c00 2019-12-07T19:34:04.887+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.8s5 epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.887+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) 2019-12-07T19:34:04.887+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:04.887+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) queued 2019-12-07T19:34:04.887+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:04.887+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 27 epoch_requested: 27 RecoveryDone) prio 255 cost 10 e27) pg 0x558e841a4000 2019-12-07T19:34:04.887+0000 7fe2c7a66700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 27 epoch_requested: 27 RecoveryDone 2019-12-07T19:34:04.887+0000 7fe2c7a66700 5 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepRecovering 2019-12-07T19:34:04.888033+0000 1 0.000042 2019-12-07T19:34:04.887+0000 7fe2c7a66700 5 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:04.887+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:04.887+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:04.906+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:04.956+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.006+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.056+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.107+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.157+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.207+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.230+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83d41400 0x558e81a38580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:05.230+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83d41400 0x558e81a38580 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:05.230+0000 7fe2e753b700 10 osd.0 27 ms_handle_authentication new session 0x558e818e9ba0 con 0x558e83d41400 entity client.admin addr 2019-12-07T19:34:05.230+0000 7fe2e753b700 10 osd.0 27 ms_handle_authentication session 0x558e818e9ba0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:05.230+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/2559427579 conn(0x558e83d41400 0x558e81a38580 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=client.4609 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:05.230+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4609 172.21.15.131:0/2559427579 1 ==== command(tid 1: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558e841994a0 con 0x558e83d41400 2019-12-07T19:34:05.230+0000 7fe2d959f700 20 osd.0 27 OSD::ms_dispatch: command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:05.230+0000 7fe2d959f700 10 osd.0 27 do_waiters -- start 2019-12-07T19:34:05.230+0000 7fe2d959f700 10 osd.0 27 do_waiters -- finish 2019-12-07T19:34:05.230+0000 7fe2d959f700 20 osd.0 27 _dispatch 0x558e841994a0 command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:05.231+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/2559427579 -- command_reply(tid 1: 0 ) v1 -- 0x558e8022fc00 con 0x558e83d41400 2019-12-07T19:34:05.251+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4609 172.21.15.131:0/2559427579 2 ==== command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558e8122de00 con 0x558e83d41400 2019-12-07T19:34:05.251+0000 7fe2d959f700 20 osd.0 27 OSD::ms_dispatch: command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:05.251+0000 7fe2d959f700 10 osd.0 27 do_waiters -- start 2019-12-07T19:34:05.251+0000 7fe2d959f700 10 osd.0 27 do_waiters -- finish 2019-12-07T19:34:05.251+0000 7fe2d959f700 20 osd.0 27 _dispatch 0x558e8122de00 command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:05.253+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/2559427579 -- command_reply(tid 2: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558e8022ee00 con 0x558e83d41400 2019-12-07T19:34:05.257+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.307+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.357+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.407+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.409+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/2559427579 conn(0x558e83d41400 msgr2=0x558e81a38580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 61 2019-12-07T19:34:05.409+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/2559427579 conn(0x558e83d41400 msgr2=0x558e81a38580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:05.409+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/2559427579 conn(0x558e83d41400 0x558e81a38580 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:05.409+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/2559427579 conn(0x558e83d41400 0x558e81a38580 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:05.409+0000 7fe2d959f700 2 osd.0 27 ms_handle_reset con 0x558e83d41400 session 0x558e818e9ba0 2019-12-07T19:34:05.452+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.3 172.21.15.131:0/37076 5 ==== osd_ping(ping e27 up_from 13 ping_stamp 2019-12-07T19:34:05.453350+0000/28.910862610s send_stamp 28.910862610s delta_ub 0.003015108s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84053a00 con 0x558e818ff800 2019-12-07T19:34:05.452+0000 7fe2e753b700 20 osd.0 27 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002682912s,0.003015108s]) 2019-12-07T19:34:05.452+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e27 up_from 13 ping_stamp 2019-12-07T19:34:05.453350+0000/28.910862610s send_stamp 28.908179698s delta_ub 18446744073.706868704s) v5 -- 0x558e8118f800 con 0x558e818ff800 2019-12-07T19:34:05.452+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.3 172.21.15.131:0/37076 5 ==== osd_ping(ping e27 up_from 13 ping_stamp 2019-12-07T19:34:05.453350+0000/28.910862610s send_stamp 28.910862610s delta_ub 0.003015108s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8a00 con 0x558e81984000 2019-12-07T19:34:05.452+0000 7fe2e6d3a700 20 osd.0 27 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002657868s,0.003015108s]) 2019-12-07T19:34:05.452+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e27 up_from 13 ping_stamp 2019-12-07T19:34:05.453350+0000/28.910862610s send_stamp 28.908204742s delta_ub 18446744073.706893748s) v5 -- 0x558e818efc00 con 0x558e81984000 2019-12-07T19:34:05.457+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.507+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.522+0000 7fe2d5281700 20 osd.0 27 reports for 0 queries 2019-12-07T19:34:05.522+0000 7fe2d5281700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.0 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558e83afae00 con 0x558e81748000 2019-12-07T19:34:05.522+0000 7fe2d5281700 20 bluestore(/var/lib/ceph/osd/ceph-0) pool_statfs pool 1 2019-12-07T19:34:05.522+0000 7fe2d5281700 10 bluestore(/var/lib/ceph/osd/ceph-0) pool_statfsstore_statfs(0x0/0x0/0x0, data 0x559000/0x537000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0) 2019-12-07T19:34:05.522+0000 7fe2d5281700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(1 pgs tid 0 v 0) v2 -- 0x558e83e14a80 con 0x558e81748000 2019-12-07T19:34:05.548+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:05.558+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.582+0000 7fe2e2ca2700 10 osd.0 27 tick_without_osd_lock 2019-12-07T19:34:05.582+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fa80000/0x40000000/0x1680000000, data 0x57394f/0x57e000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:05.582+0000 7fe2e2ca2700 25 osd.0 27 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254252+0000 last_rx_front 2019-12-07T19:34:02.254310+0000 2019-12-07T19:34:05.582+0000 7fe2e2ca2700 25 osd.0 27 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254273+0000 last_rx_front 2019-12-07T19:34:02.254232+0000 2019-12-07T19:34:05.582+0000 7fe2e2ca2700 25 osd.0 27 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254209+0000 last_rx_front 2019-12-07T19:34:02.254194+0000 2019-12-07T19:34:05.582+0000 7fe2e2ca2700 25 osd.0 27 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254409+0000 last_rx_front 2019-12-07T19:34:02.254290+0000 2019-12-07T19:34:05.582+0000 7fe2e2ca2700 25 osd.0 27 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254354+0000 last_rx_front 2019-12-07T19:34:02.254301+0000 2019-12-07T19:34:05.582+0000 7fe2e2ca2700 25 osd.0 27 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254368+0000 last_rx_front 2019-12-07T19:34:02.254333+0000 2019-12-07T19:34:05.582+0000 7fe2e2ca2700 20 osd.0 27 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:05.582+0000 7fe2e2ca2700 10 osd.0 27 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:05.582+0000 7fe2e2ca2700 20 osd.0 27 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:05.582+0000 7fe2e2ca2700 10 osd.0 27 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:05.608+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 559139 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.638+0000 7fe2e4535700 10 osd.0 27 tick 2019-12-07T19:34:05.638+0000 7fe2e4535700 10 osd.0 27 do_waiters -- start 2019-12-07T19:34:05.638+0000 7fe2e4535700 10 osd.0 27 do_waiters -- finish 2019-12-07T19:34:05.638+0000 7fe2e4535700 20 osd.0 27 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:05.652+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 25 ==== osd_map(28..28 src has 1..28) v4 ==== 982+0+0 (crc 0 0 0) 0x558e8419dd40 con 0x558e80f39000 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 osd.0 27 OSD::ms_dispatch: osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 osd.0 27 do_waiters -- start 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 osd.0 27 do_waiters -- finish 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 osd.0 27 _dispatch 0x558e8419dd40 osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.652+0000 7fe2d959f700 3 osd.0 27 handle_osd_map epochs [28,28], i have 27, src has [1,28] 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 osd.0 27 handle_osd_map got inc map for epoch 28 2019-12-07T19:34:05.652+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3ea0e826:::osdmap.27:0# 0x0~0 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1671 size 0x1671 (5745) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83dd2230 blob([0x10b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83dd20e0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af9180 blob([!~1000,0x10c000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,671]) SharedBlob(0x558e811d0d20 sbid 0x0)) need 0x1000~671 cache has 0x[1000~671] 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3ea0e826:::osdmap.27:0# 0x0~1671 = 5745 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 osd.0 27 add_map_bl 27 5745 bytes 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 osd.0 27 got_full_map 28, nothing requested 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 osd.0 27 handle_osd_map pg_num_history pg_num_history(e28 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps purged_snaps {28={}} 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 osd.0 27 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e28 [1,28] lci=[13,28]) 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e811fbb00 seq 20 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:ccbb972f:::inc_osdmap.28:0# key 0x7f7fffffffffffffffccbb97'/!inc_osdmap.28!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:05.652+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:ccbb972f:::inc_osdmap.28:0# 0x0~3aa 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1145 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:ccbb972f:::inc_osdmap.28:0# 0x0~3aa - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~3aa 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xc56 on front/back, now 0x0~1000 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fbb00 1 blobs 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x580000~1000/1000,1000,0 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x580000~1000] 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f94070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f940e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f94070 blob([0x580000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f940e0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.blob(0x558e83f94070) get_ref 0x0~3aa Blob(0x558e83f94070 blob([0x580000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f940e0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.blob(0x558e83f94070) get_ref init 0x1000, 1000 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~3aa: 0x0~3aa Blob(0x558e83f94070 blob([0x580000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x3aa) SharedBlob(0x558e83f940e0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e83f940f8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x3aa 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28d88) dirty_range mark inline shard dirty 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:ccbb972f:::inc_osdmap.28:0# 0x0~3aa = 0 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3058e826:::osdmap.28:0# key 0x7f7fffffffffffffff3058e8'&!osdmap.28!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:05.652+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3058e826:::osdmap.28:0# 0x0~1454 2019-12-07T19:34:05.652+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 100 ==== pg_query2(1.bs1 1.bs1 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e84314000 con 0x558e818fe400 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1146 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3058e826:::osdmap.28:0# 0x0~1454 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~454 2019-12-07T19:34:05.652+0000 7fe2e6d3a700 15 osd.0 27 enqueue_peering_evt 1.bs1 epoch_sent: 28 epoch_requested: 28 MQuery 1.bs1 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbac on front/back, now 0x0~1000 2019-12-07T19:34:05.652+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs1 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fbb00 2 blobs 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x581000~2000/1000,2000,0 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x581000~2000] 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f941c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f947e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f941c0 blob([0x581000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f947e0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.blob(0x558e83f941c0) get_ref 0x0~1000 Blob(0x558e83f941c0 blob([0x581000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f947e0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.blob(0x558e83f941c0) get_ref init 0x1000, 1000 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e83f941c0 blob([0x581000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f947e0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 101 ==== pg_query2(1.bs3 1.bs3 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e84314000 con 0x558e818fe400 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e83f947f8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:05.652+0000 7fe2e6d3a700 15 osd.0 27 enqueue_peering_evt 1.bs3 epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83af8e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edf810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:05.652+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83af8e00 blob([!~1000,0x582000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf810 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.blob(0x558e83af8e00) get_ref 0x1000~454 Blob(0x558e83af8e00 blob([!~1000,0x582000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e83edf810 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.blob(0x558e83af8e00) get_ref init 0x2000, 1000 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~454: 0x1000~454 Blob(0x558e83af8e00 blob([!~1000,0x582000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558e83edf810 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e83edf828 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1454 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f29008) dirty_range mark inline shard dirty 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3058e826:::osdmap.28:0# 0x0~1454 = 0 2019-12-07T19:34:05.652+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:05.652+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:05.652+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs1 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs1 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e83c78000 2019-12-07T19:34:05.652+0000 7fe2ca26b700 20 osd.0 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 28, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs1 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fbb00 0 blobs 2019-12-07T19:34:05.652+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:05.652+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:05.652+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:05.652+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.652+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e81928000 2019-12-07T19:34:05.652+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:05.652+0000 7fe2ca26b700 20 osd.0 op_wq _add_slot_waiter 1.bs3 peering, item epoch is 28, will wait on OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.652+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:05.652+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:05.653+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:05.653+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:05.653+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:05.653+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:05.653+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fbb00 0 blobs 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:05.653+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:05.653+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fbb00 cost 23623 (4 ios * 4000 + 7623 bytes) 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fbb00 onodes 0x558e80251b80,0x558e817b9180,0x558e83f28c80,0x558e83f28f00 shared_blobs 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28d88) update #-1:ccbb972f:::inc_osdmap.28:0# 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f28d88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:ccbb972f:::inc_osdmap.28:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f29008) update #-1:3058e826:::osdmap.28:0# 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f29008) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3058e826:::osdmap.28:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fbb00 allocated 0x[580000~3000] released 0x[] 2019-12-07T19:34:05.653+0000 7fe2d959f700 10 freelist allocate 0x580000~3000 2019-12-07T19:34:05.653+0000 7fe2d959f700 20 freelist _xor first_key 0x580000 last_key 0x580000 2019-12-07T19:34:05.653+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 prepare 2019-12-07T19:34:05.653+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e811fbb00 2019-12-07T19:34:05.653+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 103 ==== pg_query2(1.3s5 1.3s5 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e84314000 con 0x558e818fe400 2019-12-07T19:34:05.653+0000 7fe2e6d3a700 15 osd.0 27 enqueue_peering_evt 1.3s5 epoch_sent: 28 epoch_requested: 28 MQuery 1.3s5 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.653+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.3s5 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.653+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.653+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.3s5 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.653+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.653+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.3s5 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e81231000 2019-12-07T19:34:05.653+0000 7fe2c6263700 20 osd.0 op_wq _add_slot_waiter 1.3s5 peering, item epoch is 28, will wait on OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.3s5 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.653+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:05.653+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 101 ==== pg_notify2(1.2s0 1.2s0 (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956606539s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e28/28) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e81a58700 con 0x558e818d7000 2019-12-07T19:34:05.653+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 3(5) notify: (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956606539s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.653+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 3(5) notify: (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956606539s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.653+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 104 ==== pg_notify2(1.2s0 1.2s0 (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956942681s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e28/28) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e8404ca00 con 0x558e818fe400 2019-12-07T19:34:05.653+0000 7fe2e6d3a700 15 osd.0 27 enqueue_peering_evt 1.2s0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 4(1) notify: (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956942681s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.653+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 4(1) notify: (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956942681s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 3(5) notify: (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956606539s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 3(5) notify: (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956606539s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.654+0000 7fe2c6a64700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 28, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 3(5) notify: (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956606539s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {28=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)>} 2019-12-07T19:34:05.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 4(1) notify: (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956942681s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {28=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)>} 2019-12-07T19:34:05.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 4(1) notify: (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956942681s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.654+0000 7fe2c6a64700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 28, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 4(1) notify: (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956942681s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.654+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.654+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 102 ==== osd_map(28..28 src has 1..28) v4 ==== 982+0+0 (unknown 0 0 0) 0x558e83f2b8c0 con 0x558e818fe400 2019-12-07T19:34:05.654+0000 7fe2d4a80700 20 osd.0 27 OSD::ms_dispatch: osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.654+0000 7fe2d4a80700 10 osd.0 27 do_waiters -- start 2019-12-07T19:34:05.654+0000 7fe2d4a80700 10 osd.0 27 do_waiters -- finish 2019-12-07T19:34:05.654+0000 7fe2d4a80700 20 osd.0 27 _dispatch 0x558e83f2b8c0 osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.654+0000 7fe2d4a80700 3 osd.0 27 handle_osd_map epochs [28,28], i have 28, src has [1,28] 2019-12-07T19:34:05.654+0000 7fe2d4a80700 10 osd.0 27 no new maps here, dropping 2019-12-07T19:34:05.654+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 102 ==== osd_map(28..28 src has 1..28) v4 ==== 982+0+0 (unknown 0 0 0) 0x558e8419c6c0 con 0x558e818d7000 2019-12-07T19:34:05.654+0000 7fe2d4a80700 20 osd.0 27 OSD::ms_dispatch: osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.654+0000 7fe2d4a80700 10 osd.0 27 do_waiters -- start 2019-12-07T19:34:05.654+0000 7fe2d4a80700 10 osd.0 27 do_waiters -- finish 2019-12-07T19:34:05.654+0000 7fe2d4a80700 20 osd.0 27 _dispatch 0x558e8419c6c0 osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.654+0000 7fe2d4a80700 3 osd.0 27 handle_osd_map epochs [28,28], i have 28, src has [1,28] 2019-12-07T19:34:05.654+0000 7fe2d4a80700 10 osd.0 27 no new maps here, dropping 2019-12-07T19:34:05.655+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 aio_wait 2019-12-07T19:34:05.655+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fbb00 2019-12-07T19:34:05.655+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 io_done 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 181 ==== pg_query2(1.7s3 1.7s3 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e84242a00 con 0x558e818d7c00 2019-12-07T19:34:05.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.7s3 epoch_sent: 28 epoch_requested: 28 MQuery 1.7s3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.7s3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83f28c80 had 1 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83f28f00 had 1 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 182 ==== pg_query2(1.8s5 1.8s5 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e84242a00 con 0x558e818d7c00 2019-12-07T19:34:05.655+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3859939328 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.8s5 epoch_sent: 28 epoch_requested: 28 MQuery 1.8s5 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.8s5 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998559) bluestore 86 GiB free (0.95994), bluefs_ratio 0.0399475 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 183 ==== pg_query2(1.as3 1.as3 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e84242a00 con 0x558e818d7c00 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.as3 epoch_sent: 28 epoch_requested: 28 MQuery 1.as3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.as3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:05.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:05.655+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.as3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.655+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.655+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.as3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e8193b000 2019-12-07T19:34:05.655+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xd57fc~226c to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.655+0000 7fe2caa6c700 20 osd.0 op_wq _add_slot_waiter 1.as3 peering, item epoch is 28, will wait on OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.as3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.655+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xd57fc 0xd57fc~226c to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa57fc 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x7fc 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xa68 and padding block with 0x598 2019-12-07T19:34:05.655+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.7s3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.655+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.7s3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e8122e000 2019-12-07T19:34:05.655+0000 7fe2c425f700 20 osd.0 op_wq _add_slot_waiter 1.7s3 peering, item epoch is 28, will wait on OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.7s3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xd7a68 2019-12-07T19:34:05.655+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.655+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:05.655+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 103 ==== pg_query2(1.fs1 1.fs1 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e84242a00 con 0x558e818d7000 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000362433s (0.000122578s flush + 0.000239855s kv commit) 2019-12-07T19:34:05.655+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:05.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:05.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fbb00> 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.fs1 epoch_sent: 28 epoch_requested: 28 MQuery 1.fs1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.655+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 kv_submitted 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.fs1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fbb00 2019-12-07T19:34:05.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e811fbb00 osr 0x558e8107a500 2019-12-07T19:34:05.655+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) _discard drop 17 0x1b000~1000 2019-12-07T19:34:05.655+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) prepare_write seq 18 0x1b000~1000 crc cdd76f83 2019-12-07T19:34:05.655+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) _discard drop 17 0x2000~1000 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 104 ==== pg_query2(1.1s1 1.1s1 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e84242a00 con 0x558e818d7000 2019-12-07T19:34:05.655+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e83c6e140) prepare_write seq 18 0x2000~1000 crc f8c23e13 2019-12-07T19:34:05.655+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 15 osd.0 27 enqueue_peering_evt 1.1s1 epoch_sent: 28 epoch_requested: 28 MQuery 1.1s1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.655+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.1s1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.fs1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.655+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.fs1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e81a9b000 2019-12-07T19:34:05.655+0000 7fe2c8267700 20 osd.0 op_wq _add_slot_waiter 1.fs1 peering, item epoch is 28, will wait on OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.fs1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.1s1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.655+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:05.655+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.1s1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e819e4000 2019-12-07T19:34:05.655+0000 7fe2cb26d700 20 osd.0 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 28, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.1s1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 10 osd.0 27 _committed_osd_maps 28..28 2019-12-07T19:34:05.655+0000 7fe2d7285700 10 osd.0 27 advance to epoch 28 (<= last 28 <= newest_map 28) 2019-12-07T19:34:05.655+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:05.655+0000 7fe2d7285700 7 osd.0 28 consume_map version 28 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 identify_splits_and_merges 1.8s5 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 identify_splits_and_merges 1.9s3 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 identify_splits_and_merges 1.1s1 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 identify_splits_and_merges 1.as3 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 identify_splits_and_merges 1.2s0 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 identify_splits_and_merges 1.3s5 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 identify_splits_and_merges 1.bs1 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 identify_splits_and_merges 1.bs3 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 identify_splits_and_merges 1.4s1 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 identify_splits_and_merges 1.5s4 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 identify_splits_and_merges 1.ds5 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 identify_splits_and_merges 1.es4 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 identify_splits_and_merges 1.7s3 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 identify_splits_and_merges 1.fs1 e27 to e28 pg_nums {18=16} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 28 prune_pg_created 2019-12-07T19:34:05.655+0000 7fe2d7285700 10 osd.0:0.consume_map 28 (was 27) 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 2019-12-07T19:34:05.655+0000 7fe2d7285700 10 osd.0:1.consume_map 28 (was 27) 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 pending_peering first epoch 28 <= 28, requeueing 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {28=} 2019-12-07T19:34:05.655+0000 7fe2d7285700 10 osd.0:2.consume_map 28 (was 27) 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 pending_peering first epoch 28 <= 28, requeueing 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.as3 to_process <> waiting <> waiting_peering {28=} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 pending_peering first epoch 28 <= 28, requeueing 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.2s0 to_process <> waiting <> waiting_peering {28=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 4(1) notify: (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956942681s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)>} 2019-12-07T19:34:05.655+0000 7fe2d7285700 10 osd.0:3.consume_map 28 (was 27) 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 pending_peering first epoch 28 <= 28, requeueing 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.3s5 to_process <> waiting <> waiting_peering {28=} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 pending_peering first epoch 28 <= 28, requeueing 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {28=} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 pending_peering first epoch 28 <= 28, requeueing 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.bs3 to_process <> waiting <> waiting_peering {28=} 2019-12-07T19:34:05.655+0000 7fe2d7285700 10 osd.0:4.consume_map 28 (was 27) 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:05.655+0000 7fe2d7285700 10 osd.0:5.consume_map 28 (was 27) 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:05.655+0000 7fe2d7285700 10 osd.0:6.consume_map 28 (was 27) 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:05.655+0000 7fe2d7285700 10 osd.0:7.consume_map 28 (was 27) 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 pending_peering first epoch 28 <= 28, requeueing 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.7s3 to_process <> waiting <> waiting_peering {28=} 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 pending_peering first epoch 28 <= 28, requeueing 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.fs1 to_process <> waiting <> waiting_peering {28=} 2019-12-07T19:34:05.655+0000 7fe2d7285700 10 osd.0 28 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:05.655+0000 7fe2d7285700 15 osd.0 28 enqueue_peering_evt 1.8s5 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 15 osd.0 28 enqueue_peering_evt 1.9s3 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 15 osd.0 28 enqueue_peering_evt 1.1s1 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 15 osd.0 28 enqueue_peering_evt 1.as3 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 15 osd.0 28 enqueue_peering_evt 1.3s5 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 15 osd.0 28 enqueue_peering_evt 1.bs1 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 15 osd.0 28 enqueue_peering_evt 1.bs3 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 15 osd.0 28 enqueue_peering_evt 1.4s1 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 15 osd.0 28 enqueue_peering_evt 1.5s4 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 15 osd.0 28 enqueue_peering_evt 1.ds5 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 15 osd.0 28 enqueue_peering_evt 1.es4 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 15 osd.0 28 enqueue_peering_evt 1.7s3 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 15 osd.0 28 enqueue_peering_evt 1.fs1 epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.655+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) 2019-12-07T19:34:05.655+0000 7fe2d7285700 7 osd.0 28 activate_map version 28 2019-12-07T19:34:05.655+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.655+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.1s1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.655+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558e81a98000 2019-12-07T19:34:05.655+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.1s1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e819e4000 2019-12-07T19:34:05.655+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.655+0000 7fe2c4a60700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.655+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.1s1 27 -> 28 2019-12-07T19:34:05.655+0000 7fe2c4a60700 20 osd.0:6.update_pg_epoch 1.es4 27 -> 28 2019-12-07T19:34:05.655+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] handle_advance_map [3,0,2,4,5,1]/[3,0,2,4,5,1] -- 3/3 2019-12-07T19:34:05.655+0000 7fe2c4a60700 10 osd.0 pg_epoch: 27 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] handle_advance_map [1,3,4,5,0,2]/[1,3,4,5,0,2] -- 1/1 2019-12-07T19:34:05.655+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.655+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.655+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558e83d07000 2019-12-07T19:34:05.655+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.655+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.655+0000 7fe2c4a60700 20 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] new interval newup [1,3,4,5,0,2] newacting [1,3,4,5,0,2] 2019-12-07T19:34:05.655+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] new interval newup [3,0,2,4,5,1] newacting [3,0,2,4,5,1] 2019-12-07T19:34:05.655+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.655+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.655+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.655+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.5s4 27 -> 28 2019-12-07T19:34:05.655+0000 7fe2c4a60700 5 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.656680+0000 1 0.000042 2019-12-07T19:34:05.655+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.656683+0000 1 0.000030 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] handle_advance_map [2,1,3,4,0,5]/[2,1,3,4,0,5] -- 2/2 2019-12-07T19:34:05.656+0000 7fe2c4a60700 5 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.656697+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.656702+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.656+0000 7fe2c4a60700 5 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.656708+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] new interval newup [2,1,3,4,0,5] newacting [2,1,3,4,0,5] 2019-12-07T19:34:05.656+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.656712+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c4a60700 5 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.656+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.656+0000 7fe2c9269700 5 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.656727+0000 1 0.000032 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.656+0000 7fe2c9269700 5 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.656738+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.656+0000 7fe2c9269700 5 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.656748+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] new interval newup [1,3,4,5,0,2] newacting [1,3,4,5,0,2] 2019-12-07T19:34:05.656+0000 7fe2c9269700 5 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] new interval newup [3,0,2,4,5,1] newacting [3,0,2,4,5,1] 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]/[1,3,4,5,0,2147483647]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]/[3,0,2147483647,4,5,1]p3(0) r=1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.656+0000 7fe2c4a60700 15 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'4 active+remapped mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.656+0000 7fe2c7265700 15 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'7 active+remapped mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] new interval newup [2,1,3,4,0,5] newacting [2,1,3,4,0,5] 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [1,3,4,5,0,2](1) acting [1,3,4,5,0,2147483647](1)) up_thru 26 up_from 13 last_epoch_clean 19 interval(26-27 up [1,3,4,5,0,2](1) acting [1,3,4,5,0,2147483647](1) maybe_went_rw) : primary up 13-26 includes interval 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]/[2,2147483647,3,4,2147483647,5]p2(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'7 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [3,0,2,4,5,1](3) acting [3,0,2147483647,4,5,1](3)) up_thru 26 up_from 13 last_epoch_clean 19 interval(26-27 up [3,0,2,4,5,1](3) acting [3,0,2147483647,4,5,1](3) maybe_went_rw) : primary up 13-26 includes interval 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),6(5) intervals=([26,27] acting 0(4),1(0),3(1),4(2),5(3))) 2019-12-07T19:34:05.656+0000 7fe2c9269700 15 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]p2(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'7 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),6(2) intervals=([26,27] acting 0(1),1(5),3(0),4(3),5(4))) 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [2,1,3,4,0,5](2) acting [2,2147483647,3,4,2147483647,5](2)) : acting set is too small 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'7 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'7 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'4 active pruub 44.060997103s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 44.060997103s (mnow 29.111537006s + 14.949460097s) 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'7 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d1570] 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1))) 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'4 active pruub 44.060997103s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'7 active pruub 44.059111212s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 44.059111212s (mnow 29.111546697s + 14.947564515s) 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'4 active pruub 44.060997103s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'7 active pruub 44.059111212s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d1b90] 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'7 active pruub 44.059111212s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.656+0000 7fe2c4a60700 1 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'4 active pruub 44.060997103s@ mbc={} ps=[2~1]] start_peering_interval up [1,3,4,5,0,2] -> [1,3,4,5,0,2], acting [1,3,4,5,0,2147483647] -> [1,3,4,5,0,2], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 4 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 29.111567247s + 0s) 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.656+0000 7fe2c7265700 1 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'7 active pruub 44.059111212s@ mbc={} ps=[2~1]] start_peering_interval up [3,0,2,4,5,1] -> [3,0,2,4,5,1], acting [3,0,2147483647,4,5,1] -> [3,0,2,4,5,1], acting_primary 3(0) -> 3, up_primary 3(0) -> 3, role 1 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] release_backoffs [1:70000000::::0,1:80000000::::head) 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.656+0000 7fe2c9269700 1 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] start_peering_interval up [2,1,3,4,0,5] -> [2,1,3,4,0,5], acting [2,2147483647,3,4,2147483647,5] -> [2,1,3,4,0,5], acting_primary 2(0) -> 2, up_primary 2(0) -> 2, role -1 -> 4, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] release_backoffs [1:80000000::::0,1:90000000::::head) 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown mbc={} ps=[2~1]] release_backoffs [1:a0000000::::0,1:b0000000::::head) 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.656+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.656+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_role_change 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 osd.0 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.656+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.5s4_head options 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949460097s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947564515s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.656+0000 7fe2c4a60700 5 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949289263s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.657021+0000 1 0.000318 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.656+0000 7fe2c4a60700 5 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949289263s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.656+0000 7fe2c4a60700 5 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949289263s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.656+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947393799s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.657038+0000 1 0.000330 2019-12-07T19:34:05.656+0000 7fe2c4a60700 1 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949289263s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.656+0000 7fe2c9269700 5 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.657047+0000 1 0.000298 2019-12-07T19:34:05.656+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947393799s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.656+0000 7fe2c4a60700 5 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949289263s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.657052+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947393799s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.656+0000 7fe2c9269700 5 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.656+0000 7fe2c4a60700 5 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949289263s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.656+0000 7fe2c7265700 1 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947393799s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.656+0000 7fe2c9269700 5 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949289263s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.656+0000 7fe2c9269700 1 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.656+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947393799s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.657070+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:05.656+0000 7fe2c9269700 5 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.657077+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947393799s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.656+0000 7fe2c9269700 5 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947393799s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949289263s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 28 send_incremental_map 27 -> 28 to 0x558e818d7c00 v2:172.21.15.131:6823/37078 2019-12-07T19:34:05.656+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:ccbb972f:::inc_osdmap.28:0# 0x0~0 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~3aa size 0x3aa (938) 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f94070 blob([0x580000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x3aa) SharedBlob(0x558e83f940e0 sbid 0x0)) need 0x0~3aa cache has 0x[0~3aa] 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:ccbb972f:::inc_osdmap.28:0# 0x0~3aa = 938 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947393799s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.1s1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 28 add_map_inc_bl 28 938 bytes 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 28 send_incremental_map 27 -> 28 to 0x558e818d6000 v2:172.21.15.131:6804/37077 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947291363s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.656+0000 7fe2c4a60700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(28..28 src has 1..28) v4 -- 0x558e8405c000 con 0x558e818d7c00 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:05.656+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(28..28 src has 1..28) v4 -- 0x558e841a0000 con 0x558e818d6000 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 osd.0 28 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 27 -> 28 (shared) 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 28 send_incremental_map 27 -> 28 to 0x558e818d7000 v2:172.21.15.131:6812/37076 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 28 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 27 -> 28 (shared) 2019-12-07T19:34:05.656+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(28..28 src has 1..28) v4 -- 0x558e8123b440 con 0x558e818d7000 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 28 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 27 -> 28 (shared) 2019-12-07T19:34:05.656+0000 7fe2c4a60700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.es0 1.es0 (query:28 sent:28 1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949289263s) 4->0 ([18,27] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),6(5) intervals=([26,27] acting 0(4),1(0),3(1),4(2),5(3)))) e28/28) v1 -- 0x558e83ebb500 con 0x558e818d7c00 2019-12-07T19:34:05.656+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify2(1.5s0 1.5s0 (query:28 sent:28 1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 4->0 ([18,27] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1)))) e28/28) v1 -- 0x558e83c47c00 con 0x558e818d6000 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e811fa300 seq 20 2019-12-07T19:34:05.656+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.1s0 1.1s0 (query:28 sent:28 1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947393799s) 1->0 ([18,27] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),6(2) intervals=([26,27] acting 0(1),1(5),3(0),4(3),5(4)))) e28/28) v1 -- 0x558e83c9dc00 con 0x558e818d7000 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:05.656+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e819a1200 seq 10 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:05.656+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._biginfo' <- _biginfo 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:05.656+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.1s0 1.1s0 (query:28 sent:28 1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947291363s) 1->0 ([18,27] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),6(2) intervals=([26,27] acting 0(1),1(5),3(0),4(3),5(4)))) e28/28) v1 -- 0x558e83c9ea00 con 0x558e818d7000 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa300 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa300 onodes shared_blobs 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa300 allocated 0x[] released 0x[] 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._epoch' <- _epoch 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e811fb500 seq 22 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 5617 (1 ios * 4000 + 1617 bytes) 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 prepare 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes shared_blobs 2019-12-07T19:34:05.656+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa300 2019-12-07T19:34:05.656+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 io_done 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[] released 0x[] 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:05.656+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fb500 cost 5543 (1 ios * 4000 + 1543 bytes) 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fb500 onodes shared_blobs 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fb500 allocated 0x[] released 0x[] 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 prepare 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fb500 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 io_done 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558e819e7000 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558e8192b000 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.ds5 27 -> 28 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 27 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.9s3 27 -> 28 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.656+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 27 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,3] newacting [1,4,5,0,7,3] 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 28 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:05.656+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.657310+0000 1 0.000028 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.656+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:05.656+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:05.656+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.657320+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.657329+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.656+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.656+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.656+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.fs1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.656+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.656+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.fs1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e81a9b000 2019-12-07T19:34:05.656+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.656+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.fs1 27 -> 28 2019-12-07T19:34:05.656+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] handle_advance_map [3,0,5,4,1,2]/[3,0,5,4,1,2] -- 3/3 2019-12-07T19:34:05.656+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.656+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.656+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558e819e4000 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.656+0000 7fe2cb26d700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947291363s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.656+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] new interval newup [3,0,5,4,1,2] newacting [3,0,5,4,1,2] 2019-12-07T19:34:05.656+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:05.656+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.656+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.656+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 3(5) notify: (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956606539s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.656+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.656+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 3(5) notify: (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956606539s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.656+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.657437+0000 1 0.000031 2019-12-07T19:34:05.656+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.657447+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.657456+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.656+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.2s0 27 -> 28 2019-12-07T19:34:05.656+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.656+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.656+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] new interval newup [0,4,1,5,7,3] newacting [0,4,1,5,7,3] 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] state: Active advmap interval change, fast return 2019-12-07T19:34:05.656+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] new interval newup [3,0,5,4,1,2] newacting [3,0,5,4,1,2] 2019-12-07T19:34:05.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.656+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] new interval newup [0,4,1,5,7,3] newacting [0,4,1,5,7,3] 2019-12-07T19:34:05.656+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]/[3,0,5,4,1,2147483647]p3(0) r=1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.656+0000 7fe2c425f700 15 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'10 active+remapped mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.656+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] exit Started/Primary/Active/Recovered 2019-12-07T19:34:05.657530+0000 1 0.000468 2019-12-07T19:34:05.656+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+undersized+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] exit Started/Primary/Active 2019-12-07T19:34:05.657543+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] agent_stop 2019-12-07T19:34:05.656+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] exit Started/Primary 2019-12-07T19:34:05.657556+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [3,0,5,4,1,2](3) acting [3,0,5,4,1,2147483647](3)) up_thru 26 up_from 13 last_epoch_clean 19 interval(26-27 up [3,0,5,4,1,2](3) acting [3,0,5,4,1,2147483647](3) maybe_went_rw) : primary up 13-26 includes interval 2019-12-07T19:34:05.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:05.656+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] agent_stop 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(1),1(4),2(5),3(0),4(3),5(2),6(5) intervals=([26,27] acting 0(1),1(4),3(0),4(3),5(2))) 2019-12-07T19:34:05.656+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] exit Started 2019-12-07T19:34:05.657574+0000 0 0.000000 2019-12-07T19:34:05.656+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.656+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] enter Reset 2019-12-07T19:34:05.656+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=26 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.656+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d1570] 2019-12-07T19:34:05.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active pruub 44.059243341s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 44.059243341s (mnow 29.112288296s + 14.946955045s) 2019-12-07T19:34:05.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.656+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] new interval newup [0,4,1,5,7,3] newacting [0,4,1,5,7,3] 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active pruub 44.059243341s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active pruub 44.059243341s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.656+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]/[0,4,1,2147483647,2147483647,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.656+0000 7fe2c6a64700 10 osd.0 28 clear_ready_to_merge 1.2s0 2019-12-07T19:34:05.656+0000 7fe2c425f700 1 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active pruub 44.059243341s@ mbc={} ps=[2~1]] start_peering_interval up [3,0,5,4,1,2] -> [3,0,5,4,1,2], acting [3,0,5,4,1,2147483647] -> [3,0,5,4,1,2], acting_primary 3(0) -> 3, up_primary 3(0) -> 3, role 1 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.656+0000 7fe2c6a64700 15 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,26)/1 crt=27'12 lcod 23'10 mlcod 23'10 active+remapped mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] clear_stats 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] release_backoffs [1:f0000000::::0,MAX) 2019-12-07T19:34:05.656+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 23'10 active mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [0,4,1,5,7,3](0) acting [0,4,1,2147483647,2147483647,3](0)) : acting set is too small 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [0,4,1,5,7,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 23'10 active mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] noting past ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3))) 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.656+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 23'10 active mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.656+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 23'10 active mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.656+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.656+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 23'10 active mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] init_hb_stamps now [0x558e811d0af0,0x558e811d01c0,0x558e811d18f0,0x558e811d1b20,0x558e811d1570] 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 23'10 active pruub 44.065221680s@ mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] on_new_interval prior_readable_until_ub 44.065221680s (mnow 29.112389637s + 14.952832043s) 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.657+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 23'10 active pruub 44.065221680s@ mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 23'10 active pruub 44.065221680s@ mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 23'10 active pruub 44.065221680s@ mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:34:55.931664+0000, must = 0 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 23'10 active pruub 44.065221680s@ mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] start_peering_interval up [0,4,1,5,7,3] -> [0,4,1,5,7,3], acting [0,4,1,2147483647,2147483647,3] -> [0,4,1,5,7,3], acting_primary 0(0) -> 0, up_primary 0(0) -> 0, role 0 -> 0, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946955045s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) async=[5(3),7(4)] r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 23'10 unknown pruub 44.065221680s@ mbc={0={},1={},2={},3={},4={},5={}} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.657+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946803246s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.657757+0000 1 0.000300 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.657+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946803246s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.657+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946803246s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.657+0000 7fe2c425f700 1 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946803246s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:05.657+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946803246s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.657786+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7fe2c6a64700 15 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] publish_stats_to_osd 27:67 2019-12-07T19:34:05.657+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946803246s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946803246s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946803246s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.fs1 from 3(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] [0,4,1,2147483647,2147483647,3] -> [0,4,1,5,7,3], replicas changed 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946712057s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.657+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.657+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.fs0 1.fs0 (query:28 sent:28 1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946803246s) 1->0 ([18,27] all_participants=0(1),1(4),2(5),3(0),4(3),5(2),6(5) intervals=([26,27] acting 0(1),1(4),3(0),4(3),5(2)))) e28/28) v1 -- 0x558e80ff3100 con 0x558e818d7000 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.657869+0000 1 0.000361 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] state: transitioning to Primary 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.657887+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.fs0 1.fs0 (query:28 sent:28 1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946712057s) 1->0 ([18,27] all_participants=0(1),1(4),2(5),3(0),4(3),5(2),6(5) intervals=([26,27] acting 0(1),1(4),3(0),4(3),5(2)))) e28/28) v1 -- 0x558e83c48a00 con 0x558e818d7000 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] enter Started/Primary 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown pruub 44.065221680s@ mbc={} ps=[2~1]] enter Started/Primary/Peering 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering pruub 44.065221680s@ mbc={} ps=[2~1]] enter Started/Primary/Peering/GetInfo 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e83d4d800 seq 28 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering pruub 44.065221680s@ mbc={} ps=[2~1]] build_prior all_probe 0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering pruub 44.065221680s@ mbc={} ps=[2~1]] build_prior final: probe 0(0),1(2),3(5),4(1),5(3),5(4),7(4) down 6 blocked_by {} 2019-12-07T19:34:05.657+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering pruub 44.065221680s@ mbc={} ps=[2~1]] up_thru 27 < same_since 28, must notify monitor 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering pruub 44.065221680s@6 mbc={} ps=[2~1]] state: querying info from osd.1(2) 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering pruub 44.065221680s@6 mbc={} ps=[2~1]] state: querying info from osd.3(5) 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4d800 cost 5543 (1 ios * 4000 + 1543 bytes) 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4d800 onodes shared_blobs 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4d800 allocated 0x[] released 0x[] 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering pruub 44.065221680s@6 mbc={} ps=[2~1]] state: querying info from osd.4(1) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering pruub 44.065221680s@6 mbc={} ps=[2~1]] state: querying info from osd.5(3) 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 prepare 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering pruub 44.065221680s@6 mbc={} ps=[2~1]] state: querying info from osd.5(4) 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4d800 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 io_done 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering pruub 44.065221680s@6 mbc={} ps=[2~1]] state: querying info from osd.7(4) 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.7s3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering pruub 44.065221680s@6 mbc={} ps=[2~1]] check_prior_readable_down_osds prior_readable_down_osds osd.6 is dead as of epoch 28 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.7s3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e8122e000 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering pruub 44.065221680s@ mbc={} ps=[2~1]] empty prior_readable_down_osds, clearing ub 2019-12-07T19:34:05.657+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:05.657+0000 7fe2c6a64700 15 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] publish_stats_to_osd 27:68 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.7s3 27 -> 28 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 3(5) notify: (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956606539s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 27 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.3(5) 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956606539s) 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] new interval newup [1,4,3,0,7,2] newacting [1,4,3,0,7,2] 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:34:58.956638+0000, must = 0 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:05.657+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.658006+0000 1 0.000021 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 3 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.658013+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_query2(1.2s2 1.2s2 query(info 0'0 epoch_sent 28) e28/28) v1 -- 0x558e8402e500 con 0x558e818d7c00 2019-12-07T19:34:05.657+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.658018+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_query2(1.2s5 1.2s5 query(info 0'0 epoch_sent 28) e28/28) v1 -- 0x558e8417db80 con 0x558e818d7000 2019-12-07T19:34:05.657+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 28 send_incremental_map 27 -> 28 to 0x558e818fe400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(28..28 src has 1..28) v4 -- 0x558e84409d40 con 0x558e818fe400 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 28 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 27 -> 28 (shared) 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_query2(1.2s1 1.2s1 query(info 0'0 epoch_sent 28) e28/28) v1 -- 0x558e8417cc80 con 0x558e818fe400 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] new interval newup [1,4,3,0,7,2] newacting [1,4,3,0,7,2] 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 28 send_incremental_map 27 -> 28 to 0x558e818d6800 v2:172.21.15.109:6802/36936 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]/[1,2147483647,2147483647,0,7,2]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(28..28 src has 1..28) v4 -- 0x558e84409b00 con 0x558e818d6800 2019-12-07T19:34:05.657+0000 7fe2c425f700 15 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'9 active+remapped mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [1,4,3,0,7,2](1) acting [1,2147483647,2147483647,0,7,2](1)) : acting set is too small 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 28 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 27 -> 28 (shared) 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4))) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.2s3 1.2s3 query(info 0'0 epoch_sent 28) e28/28) v1 -- 0x558e819b6280 con 0x558e818d6800 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 28) e28/28) v1 -- 0x558e846c2c80 con 0x558e818d6800 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 28 send_incremental_map 27 -> 28 to 0x558e818fe000 v2:172.21.15.109:6826/36939 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(28..28 src has 1..28) v4 -- 0x558e844098c0 con 0x558e818fe000 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 28 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 27 -> 28 (shared) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_query2(1.2s4 1.2s4 query(info 0'0 epoch_sent 28) e28/28) v1 -- 0x558e846c2f00 con 0x558e818fe000 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e83e72900 seq 29 2019-12-07T19:34:05.657+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active pruub 44.061193946s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 44.061193946s (mnow 29.112785674s + 14.948408272s) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active pruub 44.061193946s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72900 cost 5467 (1 ios * 4000 + 1467 bytes) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72900 onodes shared_blobs 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72900 allocated 0x[] released 0x[] 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active pruub 44.061193946s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72900 prepare 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72900 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72900 io_done 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 28 queue_want_up_thru now 28 (was 26), currently 27 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 28 send_alive up_thru currently 27 want 28 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 28 send_alive want 28 2019-12-07T19:34:05.657+0000 7fe2c425f700 1 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'9 active pruub 44.061193946s@ mbc={} ps=[2~1]] start_peering_interval up [1,4,3,0,7,2] -> [1,4,3,0,7,2], acting [1,2147483647,2147483647,0,7,2] -> [1,4,3,0,7,2], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 3 -> 3, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- osd_alive(want up_thru 28 have 28) v1 -- 0x558e817b4960 con 0x558e80f39000 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] release_backoffs [1:e0000000::::0,1:f0000000::::head) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 4(1) notify: (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956942681s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 4(1) notify: (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956942681s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 4(1) notify: (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956942681s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.4(1) 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956942681s) 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:34:57.237928+0000, must = 0 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 4 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 28 queue_want_up_thru want 28 <= queued 28, currently 27 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.as3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.as3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e8193b000 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.657+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.as3 27 -> 28 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 27 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] handle_advance_map [1,4,2,0,5,3]/[1,4,2,0,5,3] -- 1/1 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] new interval newup [1,4,2,0,5,3] newacting [1,4,2,0,5,3] 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.658209+0000 1 0.000024 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948408272s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.658214+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.658219+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.657+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948286593s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.658229+0000 1 0.000190 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.657+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948286593s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] new interval newup [1,4,2,0,5,3] newacting [1,4,2,0,5,3] 2019-12-07T19:34:05.657+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948286593s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]/[1,2147483647,2,0,5,2147483647]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.657+0000 7fe2c425f700 1 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948286593s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.657+0000 7fe2c6a64700 15 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'2 active+remapped mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.657+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948286593s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.658255+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [1,4,2,0,5,3](1) acting [1,2147483647,2,0,5,2147483647](1)) : acting set is too small 2019-12-07T19:34:05.657+0000 7fe2c425f700 5 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948286593s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948286593s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948286593s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.7s3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948214015s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(3),1(0),2(2),3(5),4(1),4(5),5(4),6(1) intervals=([18,23] acting 0(3),1(0),2(2),4(5),5(4),6(1))) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:05.657+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:28 sent:28 1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948286593s) 3->0 ([18,27] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e28/28) v1 -- 0x558e846f8000 con 0x558e818d7c00 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'2 active pruub 44.064869888s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 44.064869888s (mnow 29.113008099s + 14.951861789s) 2019-12-07T19:34:05.657+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:28 sent:28 1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948214015s) 3->0 ([18,27] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e28/28) v1 -- 0x558e846f8700 con 0x558e818d7c00 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'2 active pruub 44.064869888s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e83d4db00 seq 30 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'2 active pruub 44.064869888s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.657+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'2 active pruub 44.064869888s@ mbc={} ps=[2~1]] start_peering_interval up [1,4,2,0,5,3] -> [1,4,2,0,5,3], acting [1,2147483647,2,0,5,2147483647] -> [1,4,2,0,5,3], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 3 -> 3, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4db00 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4db00 onodes shared_blobs 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4db00 allocated 0x[] released 0x[] 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4db00 prepare 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4db00 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4db00 io_done 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] release_backoffs [1:50000000::::0,1:60000000::::head) 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558e81a9b000 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946712057s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.657+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 6 submitting 6 deferred done 0 stable 0 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1432 < min_flush_size 524288 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2741 < min_flush_size 524288 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4128 < min_flush_size 524288 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5515 < min_flush_size 524288 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 6826 < min_flush_size 524288 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8117 < min_flush_size 524288 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:05.657+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8140 < min_flush_size 524288 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8140 < min_flush_size 524288 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xd7a68~1fcc to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xd7a68 0xd7a68~1fcc to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa7a68 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xa68 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xa34 and padding block with 0x5cc 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951861789s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951688931s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.658494+0000 1 0.000239 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951688931s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951688931s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951688931s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951688931s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.658523+0000 0 0.000000 2019-12-07T19:34:05.657+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951688931s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951688931s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951688931s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.as3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951614038s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.657+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558e8122e000 2019-12-07T19:34:05.657+0000 7fe2c8267700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948214015s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.as0 1.as0 (query:28 sent:28 1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951688931s) 3->0 ([18,27] all_participants=0(3),1(0),2(2),3(5),4(1),4(5),5(4),6(1) intervals=([18,23] acting 0(3),1(0),2(2),4(5),5(4),6(1)))) e28/28) v1 -- 0x558e8122b100 con 0x558e818d7c00 2019-12-07T19:34:05.657+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.as0 1.as0 (query:28 sent:28 1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951614038s) 3->0 ([18,27] all_participants=0(3),1(0),2(2),3(5),4(1),4(5),5(4),6(1) intervals=([18,23] acting 0(3),1(0),2(2),4(5),5(4),6(1)))) e28/28) v1 -- 0x558e81229500 con 0x558e818d7c00 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e83e72f00 seq 16 2019-12-07T19:34:05.657+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72f00 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72f00 onodes shared_blobs 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72f00 allocated 0x[] released 0x[] 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xd9a34 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72f00 prepare 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72f00 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72f00 io_done 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 6 cleaned 0 in 0.000274287s (0.000000125s flush + 0.000274162s kv commit) 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558e8193b000 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951614038s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:05.657+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:05.657+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.658+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:05.658+0000 7fe2c6a64700 10 osd.0 28 queue_want_up_thru want 28 <= queued 28, currently 27 2019-12-07T19:34:05.658+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.657+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.658+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xd9a34~522 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.658+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xd9a34 0xd9a34~522 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa9a34 2019-12-07T19:34:05.658+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xa34 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xf56 and padding block with 0xaa 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e81928000 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa300,0x558e819a1200,0x558e811fb500,0x558e83d4d800,0x558e83e72900,0x558e83d4db00> 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 kv_submitted 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa300 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 finishing 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa300 onodes 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa300 done 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa300 [] 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs3 27 -> 28 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 kv_submitted 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fb500 2019-12-07T19:34:05.658+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 finishing 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fb500 onodes 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb500 done 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:05.658+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fb500 [] 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 kv_submitted 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4d800 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 finishing 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4d800 onodes 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4d800 done 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4d800 [] 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72900 kv_submitted 2019-12-07T19:34:05.658+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] new interval newup [4,0,2,7,1,5] newacting [4,0,2,7,1,5] 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72900 2019-12-07T19:34:05.658+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72900 finishing 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72900 onodes 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72900 done 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72900 [] 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4db00 kv_submitted 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4db00 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4db00 finishing 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4db00 onodes 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4db00 done 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4db00 [] 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.658781+0000 2 0.000068 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.658792+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.658801+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xd9f56 2019-12-07T19:34:05.658+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.658+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.658+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000172985s (0.000000088s flush + 0.000172897s kv commit) 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e72f00> 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72f00 kv_submitted 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72f00 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72f00 finishing 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72f00 onodes 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72f00 done 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72f00 [] 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] new interval newup [4,0,2,7,1,5] newacting [4,0,2,7,1,5] 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.658+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:05.658+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.658+0000 7fe2ca26b700 15 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active+remapped mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [4,0,2,7,1,5](4) acting [4,2147483647,2,2147483647,1,5](4)) : acting set is too small 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=26) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] init_hb_stamps now [] 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 29.113611613s + 0s) 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.658+0000 7fe2ca26b700 1 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] start_peering_interval up [4,0,2,7,1,5] -> [4,0,2,7,1,5], acting [4,2147483647,2,2147483647,1,5] -> [4,0,2,7,1,5], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role -1 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] release_backoffs [1:d0000000::::0,1:e0000000::::head) 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_role_change 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.658+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs3_head options 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.659067+0000 1 0.000270 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.658+0000 7fe2ca26b700 1 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.659093+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.bs3 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] update_history prior_readable_until_ub 44.069625289s (mnow 29.113825151s + 14.955800138s) 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:05.658+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:28 sent:28 1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) 3->0 ([18,27] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e28/28) v1 -- 0x558e8404d100 con 0x558e818fe400 2019-12-07T19:34:05.658+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:28 sent:28 1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s) 3->0 ([18,27] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e28/28) v1 -- 0x558e8404b500 con 0x558e818fe400 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e8417a300 seq 29 2019-12-07T19:34:05.658+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._biginfo' <- _biginfo 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:05.658+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417a300 cost 6516 (1 ios * 4000 + 2516 bytes) 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417a300 onodes shared_blobs 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417a300 allocated 0x[] released 0x[] 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 prepare 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417a300 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 io_done 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs1 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.bs1 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e83c78000 2019-12-07T19:34:05.658+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:34:05.658+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2314 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2314 < min_flush_size 524288 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs1 27 -> 28 2019-12-07T19:34:05.658+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.658+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xd9f56~90a to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.658+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xd9f56 0xd9f56~90a to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xa9f56 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xf56 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x860 and padding block with 0x7a0 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] new interval newup [4,0,2,7,1,5] newacting [4,0,2,7,1,5] 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.659339+0000 1 0.000038 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.659349+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.659357+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] new interval newup [4,0,2,7,1,5] newacting [4,0,2,7,1,5] 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xda860 2019-12-07T19:34:05.658+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.658+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:05.658+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]/[4,2147483647,2,2147483647,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000153918s (0.000000094s flush + 0.000153824s kv commit) 2019-12-07T19:34:05.658+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:05.658+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 105 ==== pg_notify2(1.2s0 1.2s0 (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951500973s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e28/28) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e8404ca00 con 0x558e818fe400 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417a300> 2019-12-07T19:34:05.658+0000 7fe2ca26b700 15 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 kv_submitted 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417a300 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 finishing 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417a300 onodes 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a300 done 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417a300 [] 2019-12-07T19:34:05.658+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.658+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:05.658+0000 7fe2e6d3a700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 4(1) notify: (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951500973s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.658+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 4(1) notify: (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951500973s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [4,0,2,7,1,5](4) acting [4,2147483647,2,2147483647,1,5](4)) : acting set is too small 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1))) 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d0af0] 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 29.114186412s + 0s) 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.658+0000 7fe2ca26b700 1 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] start_peering_interval up [4,0,2,7,1,5] -> [4,0,2,7,1,5], acting [4,2147483647,2,2147483647,1,5] -> [4,0,2,7,1,5], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role -1 -> 1, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] release_backoffs [1:d0000000::::0,1:e0000000::::head) 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.658+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.8s5 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.658+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.658+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.8s5 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e841a4000 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_role_change 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.658+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs1_head options 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.658+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.658+0000 7fe2cba6e700 20 osd.0:0.update_pg_epoch 1.8s5 27 -> 28 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.658+0000 7fe2cba6e700 10 osd.0 pg_epoch: 27 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] handle_advance_map [1,4,5,2,3,0]/[1,4,5,2,3,0] -- 1/1 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.658+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.658+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.658+0000 7fe2cba6e700 20 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] new interval newup [1,4,5,2,3,0] newacting [1,4,5,2,3,0] 2019-12-07T19:34:05.658+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.659643+0000 1 0.000289 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.658+0000 7fe2cba6e700 5 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.659652+0000 1 0.000027 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.658+0000 7fe2ca26b700 1 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.658+0000 7fe2cba6e700 5 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.659664+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.659671+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7fe2cba6e700 5 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.659673+0000 0 0.000000 2019-12-07T19:34:05.658+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.658+0000 7fe2cba6e700 5 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.658+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.bs1 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] new interval newup [1,4,5,2,3,0] newacting [1,4,5,2,3,0] 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]/[1,4,5,2,3,2147483647]p1(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.659+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:28 sent:28 1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 1->0 ([18,27] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e28/28) v1 -- 0x558e8404a000 con 0x558e818fe400 2019-12-07T19:34:05.659+0000 7fe2cba6e700 15 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]p1(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.659+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:28 sent:28 1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 1->0 ([18,27] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e28/28) v1 -- 0x558e8404a700 con 0x558e818fe400 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e811faf00 seq 9 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [1,4,5,2,3,0](1) acting [1,4,5,2,3,2147483647](1)) up_thru 26 up_from 13 last_epoch_clean 19 interval(26-27 up [1,4,5,2,3,0](1) acting [1,4,5,2,3,2147483647](1) maybe_went_rw) : primary up 13-26 includes interval 2019-12-07T19:34:05.659+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._biginfo' <- _biginfo 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=26) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([26,27] acting 1(0),2(3),3(4),4(1),5(2))) 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._epoch' <- _epoch 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811faf00 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811faf00 onodes shared_blobs 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811faf00 allocated 0x[] released 0x[] 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 prepare 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 29.114499585s + 0s) 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811faf00 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 io_done 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.3s5 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1291 < min_flush_size 524288 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.3s5 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e81231000 2019-12-07T19:34:05.659+0000 7fe2cba6e700 1 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] start_peering_interval up [1,4,5,2,3,0] -> [1,4,5,2,3,0], acting [1,4,5,2,3,2147483647] -> [1,4,5,2,3,0], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role -1 -> 5, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1314 < min_flush_size 524288 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown mbc={} ps=[2~1]] release_backoffs [1:10000000::::0,1:20000000::::head) 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.3s5 27 -> 28 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xda860~522 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xda860 0xda860~522 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xaa860 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 27 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x860 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd82 and padding block with 0x27e 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] new interval newup [4,1,2,5,7,0] newacting [4,1,2,5,7,0] 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] state: should_restart_peering, transitioning to Reset 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.659+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] exit Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:05.659891+0000 1 0.000034 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.659+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] exit Started/ReplicaActive 2019-12-07T19:34:05.659901+0000 0 0.000000 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.659+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] exit Started 2019-12-07T19:34:05.659910+0000 0 0.000000 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_role_change 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.659+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] enter Reset 2019-12-07T19:34:05.659+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.8s5_head options 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=26 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] Clearing blocked outgoing recovery messages 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] Not blocking outgoing recovery messages 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xdad82 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] new interval newup [4,1,2,5,7,0] newacting [4,1,2,5,7,0] 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000137192s (0.000000101s flush + 0.000137091s kv commit) 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811faf00> 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.659+0000 7fe2cba6e700 5 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.659966+0000 1 0.000298 2019-12-07T19:34:05.659+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 kv_submitted 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811faf00 2019-12-07T19:34:05.659+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 finishing 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811faf00 onodes 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811faf00 done 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]/[4,1,2,5,7,2147483647]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.659+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811faf00 [] 2019-12-07T19:34:05.659+0000 7fe2cba6e700 5 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.659+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:05.659+0000 7fe2cba6e700 5 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.659+0000 7fe2ca26b700 15 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]p4(0) r=-1 lpr=28 pi=[18,26)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.659+0000 7fe2cba6e700 1 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.659+0000 7fe2cba6e700 5 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.659993+0000 0 0.000000 2019-12-07T19:34:05.659+0000 7fe2cba6e700 5 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [4,1,2,5,7,0](4) acting [4,1,2,5,7,2147483647](4)) up_thru 26 up_from 13 last_epoch_clean 19 interval(26-27 up [4,1,2,5,7,0](4) acting [4,1,2,5,7,2147483647](4) maybe_went_rw) : primary up 13-26 includes interval 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([26,27] acting 1(1),2(2),4(0),5(3),7(4))) 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.8s5 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d0af0] 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 0s (mnow 29.114734930s + 0s) 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] update_history prior_readable_until_ub 44.065109905s (mnow 29.114728461s + 14.950381444s) 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7fe2ca26b700 1 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] start_peering_interval up [4,1,2,5,7,0] -> [4,1,2,5,7,0], acting [4,1,2,5,7,2147483647] -> [4,1,2,5,7,0], acting_primary 4(0) -> 4, up_primary 4(0) -> 4, role -1 -> 5, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.659+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.8s0 1.8s0 (query:28 sent:28 1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 5->0 ([18,27] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([26,27] acting 1(0),2(3),3(4),4(1),5(2)))) e28/28) v1 -- 0x558e80ff3800 con 0x558e818d7c00 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] release_backoffs [1:c0000000::::0,1:d0000000::::head) 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.659+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.8s0 1.8s0 (query:28 sent:28 1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s) 5->0 ([18,27] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([26,27] acting 1(0),2(3),3(4),4(1),5(2)))) e28/28) v1 -- 0x558e846faa00 con 0x558e818d7c00 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e81a0b800 seq 23 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.659+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._biginfo' <- _biginfo 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._epoch' <- _epoch 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:05.659+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b800 cost 6730 (1 ios * 4000 + 2730 bytes) 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b800 onodes shared_blobs 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b800 allocated 0x[] released 0x[] 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_role_change 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 prepare 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b800 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 io_done 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.659+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.3s5_head options 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558e841a4000 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.659+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.659+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.659+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558e81938000 2019-12-07T19:34:05.659+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.660200+0000 1 0.000286 2019-12-07T19:34:05.659+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.659+0000 7fe2c5a62700 10 osd.0 pg_epoch: 27 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_advance_map: 28 2019-12-07T19:34:05.659+0000 7fe2c5a62700 20 osd.0:4.update_pg_epoch 1.4s1 27 -> 28 2019-12-07T19:34:05.659+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.659+0000 7fe2ca26b700 1 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.659+0000 7fe2c5a62700 10 osd.0 pg_epoch: 27 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:05.659+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.660229+0000 0 0.000000 2019-12-07T19:34:05.659+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7fe2ca26b700 5 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.3s5 from 4(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:28 sent:28 1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 5->0 ([18,27] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([26,27] acting 1(1),2(2),4(0),5(3),7(4)))) e28/28) v1 -- 0x558e84704000 con 0x558e818fe400 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2505 < min_flush_size 524288 2019-12-07T19:34:05.659+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:28 sent:28 1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 5->0 ([18,27] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([26,27] acting 1(1),2(2),4(0),5(3),7(4)))) e28/28) v1 -- 0x558e84704700 con 0x558e818fe400 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e811fa600 seq 9 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2528 < min_flush_size 524288 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2528 < min_flush_size 524288 2019-12-07T19:34:05.659+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._biginfo' <- _biginfo 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xdad82~9e0 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._epoch' <- _epoch 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xdad82 0xdad82~9e0 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xaad82 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd82 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x762 and padding block with 0x89e 2019-12-07T19:34:05.659+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa600 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa600 onodes shared_blobs 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa600 allocated 0x[] released 0x[] 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 prepare 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa600 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 io_done 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558e83c78000 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558e81928000 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.659+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.659+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 NullEvt) prio 255 cost 10 e28) pg 0x558e81231000 2019-12-07T19:34:05.659+0000 7fe2c6263700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.659+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:05.659+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xdb762 2019-12-07T19:34:05.659+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000251700s (0.000000096s flush + 0.000251604s kv commit) 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1269 < min_flush_size 524288 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xdb762~50c to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.659+0000 7fe2cc26f700 20 osd.0 28 agent_entry empty queue 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0b800> 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.659+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 kv_submitted 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b800 2019-12-07T19:34:05.659+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 finishing 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b800 onodes 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 done 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:05.659+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b800 [] 2019-12-07T19:34:05.659+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.659+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:05.659+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:05.659+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xdb762 0xdb762~50c to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xab762 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x762 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.659+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc6e and padding block with 0x392 2019-12-07T19:34:05.660+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xdbc6e 2019-12-07T19:34:05.659+0000 7fe2c5a62700 10 osd.0 pg_epoch: 28 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:05.661+0000 7fe2c5a62700 10 osd.0 pg_epoch: 28 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.661+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:34:05.661+0000 7fe2c5a62700 10 osd.0 pg_epoch: 28 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.661+0000 7fe2c5a62700 20 osd.0 pg_epoch: 28 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.661+0000 7fe2c5a62700 20 osd.0 pg_epoch: 28 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.661+0000 7fe2c5a62700 20 osd.0 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.661+0000 7fe2c5a62700 10 osd.0 pg_epoch: 28 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.661+0000 7fe2c5a62700 10 osd.0 pg_epoch: 28 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.661+0000 7fe2c5a62700 20 osd.0 pg_epoch: 28 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 23 while current is 28 2019-12-07T19:34:05.661+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:05.661+0000 7fe2c5a62700 10 osd.0 pg_epoch: 28 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.661+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:05.661+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.660+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 46 ==== pg_notify2(1.2s0 1.2s0 (query:28 sent:28 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e28/28) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e83c9f100 con 0x558e818d6800 2019-12-07T19:34:05.661+0000 7fe2e753b700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(3) notify: (query:28 sent:28 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.661+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(3) notify: (query:28 sent:28 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.660+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:05.656+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] state: Reset advmap 2019-12-07T19:34:05.661+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] new interval newup [1,4,5,0,7,3] newacting [1,4,5,0,7,3] 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] state: should restart peering, calling start_peering_interval again 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]/[1,4,5,0,7,2147483647]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] set_last_peering_reset 28 2019-12-07T19:34:05.662+0000 7fe2c7265700 15 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,26)/1 luod=0'0 crt=27'8 active+remapped mbc={} ps=[2~1]] clear_stats 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] start_peering_interval: check_new_interval output: check_new_interval interval(26-27 up [1,4,5,0,7,3](1) acting [1,4,5,0,7,2147483647](1)) up_thru 26 up_from 13 last_epoch_clean 19 interval(26-27 up [1,4,5,0,7,3](1) acting [1,4,5,0,7,2147483647](1) maybe_went_rw) : primary up 13-26 includes interval 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=26) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] noting past ([18,27] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([26,27] acting 0(3),1(0),4(1),5(2),7(4))) 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] on_new_interval 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] on_new_interval checking missing set deletes flag. missing = missing(0 may_include_deletes = 1) 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] init_hb_stamps now [0x558e811d01c0] 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active pruub 44.060646370s@ mbc={} ps=[2~1]] on_new_interval prior_readable_until_ub 44.060646370s (mnow 29.117474570s + 14.943171800s) 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active pruub 44.060646370s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active pruub 44.060646370s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:05.662+0000 7fe2c7265700 1 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active pruub 44.060646370s@ mbc={} ps=[2~1]] start_peering_interval up [1,4,5,0,7,3] -> [1,4,5,0,7,3], acting [1,4,5,0,7,2147483647] -> [1,4,5,0,7,3], acting_primary 1(0) -> 1, up_primary 1(0) -> 1, role 3 -> 3, features acting 4540138292836696063 upacting 4540138292836696063 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] clear_primary_state 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] release_backoffs [1:90000000::::0,1:a0000000::::head) 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] on_change_cleanup 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:05.662+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 osd.0 28 advance_pg new pool opts old pool opts 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] handle_activate_map: 28 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943171800s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:05.662+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943013621s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] exit Reset 2019-12-07T19:34:05.662949+0000 1 0.005617 2019-12-07T19:34:05.662+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943013621s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] enter Started 2019-12-07T19:34:05.662+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943013621s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] enter Start 2019-12-07T19:34:05.662+0000 7fe2c7265700 1 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943013621s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] state: transitioning to Stray 2019-12-07T19:34:05.662+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943013621s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] exit Start 2019-12-07T19:34:05.662979+0000 0 0.000000 2019-12-07T19:34:05.662+0000 7fe2c7265700 5 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943013621s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] enter Started/Stray 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943013621s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 27 while current is 28 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943013621s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 NullEvt 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:05.662+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:28 sent:28 1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943013621s) 3->0 ([18,27] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([26,27] acting 0(3),1(0),4(1),5(2),7(4)))) e28/28) v1 -- 0x558e8404ca00 con 0x558e818d7c00 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83d4d800 seq 22 2019-12-07T19:34:05.662+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4d800 cost 5543 (1 ios * 4000 + 1543 bytes) 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4d800 onodes shared_blobs 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4d800 allocated 0x[] released 0x[] 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 prepare 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4d800 2019-12-07T19:34:05.662+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 io_done 2019-12-07T19:34:05.662+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:05.662+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:05.660+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 4(1) notify: (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951500973s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.660+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 184 ==== osd_map(28..28 src has 1..28) v4 ==== 982+0+0 (unknown 0 0 0) 0x558e841a0240 con 0x558e818d7c00 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7fe2d4a80700 20 osd.0 28 OSD::ms_dispatch: osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.662+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- start 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 4(1) notify: (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951500973s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.662+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- finish 2019-12-07T19:34:05.662+0000 7fe2d4a80700 20 osd.0 28 _dispatch 0x558e841a0240 osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.662+0000 7fe2d4a80700 3 osd.0 28 handle_osd_map epochs [28,28], i have 28, src has [1,28] 2019-12-07T19:34:05.662+0000 7fe2d4a80700 10 osd.0 28 no new maps here, dropping 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 4(1) notify: (query:28 sent:28 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951500973s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] got dup osd.4(1) info 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951500973s), identical to ours 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 28 queue_want_up_thru want 28 <= queued 28, currently 27 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.662+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.662+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 48 ==== pg_notify2(1.2s0 1.2s0 (query:28 sent:28 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e28/28) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e83c9f100 con 0x558e818d6800 2019-12-07T19:34:05.662+0000 7fe2e753b700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(4) notify: (query:28 sent:28 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.662+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(4) notify: (query:28 sent:28 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.662+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 47 ==== osd_map(28..28 src has 1..28) v4 ==== 982+0+0 (unknown 0 0 0) 0x558e8419d680 con 0x558e818d6800 2019-12-07T19:34:05.662+0000 7fe2d4a80700 20 osd.0 28 OSD::ms_dispatch: osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.662+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- start 2019-12-07T19:34:05.662+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- finish 2019-12-07T19:34:05.662+0000 7fe2d4a80700 20 osd.0 28 _dispatch 0x558e8419d680 osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.662+0000 7fe2d4a80700 3 osd.0 28 handle_osd_map epochs [28,28], i have 28, src has [1,28] 2019-12-07T19:34:05.662+0000 7fe2d4a80700 10 osd.0 28 no new maps here, dropping 2019-12-07T19:34:05.662+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 28 ==== osd_map(28..28 src has 1..28) v4 ==== 982+0+0 (unknown 0 0 0) 0x558e841a0d80 con 0x558e818fe000 2019-12-07T19:34:05.662+0000 7fe2d4a80700 20 osd.0 28 OSD::ms_dispatch: osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.662+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- start 2019-12-07T19:34:05.662+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- finish 2019-12-07T19:34:05.662+0000 7fe2d4a80700 20 osd.0 28 _dispatch 0x558e841a0d80 osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.662+0000 7fe2d4a80700 3 osd.0 28 handle_osd_map epochs [28,28], i have 28, src has [1,28] 2019-12-07T19:34:05.662+0000 7fe2d4a80700 10 osd.0 28 no new maps here, dropping 2019-12-07T19:34:05.662+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 29 ==== pg_notify2(1.2s0 1.2s0 (query:28 sent:28 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e28/28) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e83c9f100 con 0x558e818fe000 2019-12-07T19:34:05.662+0000 7fe2e753b700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 7(4) notify: (query:28 sent:28 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.662+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 7(4) notify: (query:28 sent:28 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.662+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 30 ==== pg_info2(1.4s1 1.4s1 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e28/28) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c9f100 con 0x558e818fe000 2019-12-07T19:34:05.662+0000 7fe2e753b700 15 osd.0 28 enqueue_peering_evt 1.4s1 epoch_sent: 28 epoch_requested: 28 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s) 2019-12-07T19:34:05.662+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e28) 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e28) pg 0x558e81938000 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 28 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s) 2019-12-07T19:34:05.662+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 35 ==== pg_query2(1.5s4 1.5s4 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e8417d900 con 0x558e818d6000 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 28 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:05.662+0000 7fe2e753b700 15 osd.0 28 enqueue_peering_evt 1.5s4 epoch_sent: 28 epoch_requested: 28 MQuery 1.5s4 from 2(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.662+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.5s4 from 2(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:34:05.662+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e8195ef00 seq 10 2019-12-07T19:34:05.662+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.5s4 from 2(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.662+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.5s4 from 2(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e83d07000 2019-12-07T19:34:05.662+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 36 ==== osd_map(28..28 src has 1..28) v4 ==== 982+0+0 (unknown 0 0 0) 0x558e841a0000 con 0x558e818d6000 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:34:05.662+0000 7fe2d4a80700 20 osd.0 28 OSD::ms_dispatch: osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.662+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- start 2019-12-07T19:34:05.662+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- finish 2019-12-07T19:34:05.662+0000 7fe2d4a80700 20 osd.0 28 _dispatch 0x558e841a0000 osd_map(28..28 src has 1..28) v4 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:05.662+0000 7fe2d4a80700 3 osd.0 28 handle_osd_map epochs [28,28], i have 28, src has [1,28] 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:05.662+0000 7fe2d4a80700 10 osd.0 28 no new maps here, dropping 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:05.662+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.5s4 from 2(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195ef00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195ef00 onodes shared_blobs 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195ef00 allocated 0x[] released 0x[] 2019-12-07T19:34:05.662+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946296912s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946296912s 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 prepare 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195ef00 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 io_done 2019-12-07T19:34:05.662+0000 7fe2c9269700 20 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946296912s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] update_history prior_readable_until_ub 44.064407826s (mnow 29.118110914s + 14.946296912s) 2019-12-07T19:34:05.662+0000 7fe2c9269700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946296912s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.662+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(3) notify: (query:28 sent:28 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.662+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(3) notify: (query:28 sent:28 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(3) notify: (query:28 sent:28 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.662+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify2(1.5s0 1.5s0 (query:28 sent:28 1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946296912s) 4->0 ([18,27] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1)))) e28/28) v1 -- 0x558e84704000 con 0x558e818d6000 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.5(3) 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 2019-12-07T19:34:05.662+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:00.724731+0000, must = 0 2019-12-07T19:34:05.662+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e819a0c00 seq 11 2019-12-07T19:34:05.662+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:05.662+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:05.662+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:05.662+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0c00 cost 5362 (1 ios * 4000 + 1362 bytes) 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:05.662+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0c00 onodes shared_blobs 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:05.662+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 28 queue_want_up_thru want 28 <= queued 28, currently 27 2019-12-07T19:34:05.662+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 prepare 2019-12-07T19:34:05.662+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0c00 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 io_done 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(4) notify: (query:28 sent:28 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(4) notify: (query:28 sent:28 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(4) notify: (query:28 sent:28 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.5(4) 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:07.814577+0000, must = 0 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] osd.5(4) has stray content: 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 5 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 28 queue_want_up_thru want 28 <= queued 28, currently 27 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 7(4) notify: (query:28 sent:28 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 7(4) notify: (query:28 sent:28 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 7(4) notify: (query:28 sent:28 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.7(4) 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:35:23.930658+0000, must = 0 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 7 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:05.662+0000 7fe2c6a64700 10 osd.0 28 queue_want_up_thru want 28 <= queued 28, currently 27 2019-12-07T19:34:05.662+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.662+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 49 ==== pg_notify2(1.2s0 1.2s0 (query:28 sent:28 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e28/28) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e83c47c00 con 0x558e818d6800 2019-12-07T19:34:05.663+0000 7fe2e753b700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(3) notify: (query:28 sent:28 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.663+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(3) notify: (query:28 sent:28 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.663+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 50 ==== pg_notify2(1.2s0 1.2s0 (query:28 sent:28 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e28/28) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e83c47c00 con 0x558e818d6800 2019-12-07T19:34:05.663+0000 7fe2e753b700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(4) notify: (query:28 sent:28 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.663+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(4) notify: (query:28 sent:28 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.663+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 31 ==== pg_notify2(1.2s0 1.2s0 (query:28 sent:28 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e28/28) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e83c47c00 con 0x558e818fe000 2019-12-07T19:34:05.663+0000 7fe2e753b700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 7(4) notify: (query:28 sent:28 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.663+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 7(4) notify: (query:28 sent:28 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(3) notify: (query:28 sent:28 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(3) notify: (query:28 sent:28 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(3) notify: (query:28 sent:28 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] got dup osd.5(3) info 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s), identical to ours 2019-12-07T19:34:05.663+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:05.663+0000 7fe2caa6c700 10 osd.0 28 queue_want_up_thru want 28 <= queued 28, currently 27 2019-12-07T19:34:05.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(4) notify: (query:28 sent:28 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(4) notify: (query:28 sent:28 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 5(4) notify: (query:28 sent:28 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] got dup osd.5(4) info 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s), identical to ours 2019-12-07T19:34:05.663+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:05.663+0000 7fe2caa6c700 10 osd.0 28 queue_want_up_thru want 28 <= queued 28, currently 27 2019-12-07T19:34:05.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 7(4) notify: (query:28 sent:28 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 7(4) notify: (query:28 sent:28 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 7(4) notify: (query:28 sent:28 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] got dup osd.7(4) info 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.952832043s), identical to ours 2019-12-07T19:34:05.663+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:05.663+0000 7fe2caa6c700 10 osd.0 28 queue_want_up_thru want 28 <= queued 28, currently 27 2019-12-07T19:34:05.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.660+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000016 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.004196230s (0.000000087s flush + 0.004196143s kv commit) 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1387 < min_flush_size 524288 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2521 < min_flush_size 524288 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3714 < min_flush_size 524288 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3737 < min_flush_size 524288 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3737 < min_flush_size 524288 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xdbc6e~e99 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xdbc6e 0xdbc6e~e99 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xabc6e 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc6e 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb07 and padding block with 0x4f9 2019-12-07T19:34:05.664+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 37 ==== pg_info2(1.ds5 1.ds5 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e28/28) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e84704000 con 0x558e818d6000 2019-12-07T19:34:05.664+0000 7fe2e753b700 15 osd.0 28 enqueue_peering_evt 1.ds5 epoch_sent: 28 epoch_requested: 28 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:05.664+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e28) 2019-12-07T19:34:05.664+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.664+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.664+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.664+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e28) pg 0x558e819e7000 2019-12-07T19:34:05.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 28 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:05.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 28 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:05.664+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:05.664+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:34:05.664+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e8417af00 seq 17 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:05.664+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa600> 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 kv_submitted 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa600 2019-12-07T19:34:05.664+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 finishing 2019-12-07T19:34:05.664+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa600 onodes 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa600 done 2019-12-07T19:34:05.664+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa600 [] 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.664+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:34:05.664+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417af00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:05.664+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417af00 onodes shared_blobs 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:05.664+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417af00 allocated 0x[] released 0x[] 2019-12-07T19:34:05.664+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 prepare 2019-12-07T19:34:05.664+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417af00 2019-12-07T19:34:05.664+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 io_done 2019-12-07T19:34:05.664+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xdcb07 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 3 cleaned 0 in 0.000455123s (0.000000083s flush + 0.000455040s kv commit) 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4d800,0x558e8195ef00,0x558e819a0c00> 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 kv_submitted 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4d800 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 finishing 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4d800 onodes 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4d800 done 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4d800 [] 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 kv_submitted 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195ef00 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 finishing 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195ef00 onodes 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195ef00 done 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df900 q now empty 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195ef00 [] 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 kv_submitted 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xdcb07~485 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0c00 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 finishing 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xdcb07 0xdcb07~485 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0c00 onodes 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0c00 done 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0c00 [] 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xacb07 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb07 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xf8c and padding block with 0x74 2019-12-07T19:34:05.664+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xdcf8c 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:05.664+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000098699s (0.000000080s flush + 0.000098619s kv commit) 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:05.664+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417af00> 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 kv_submitted 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417af00 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 finishing 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417af00 onodes 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417af00 done 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417af00 [] 2019-12-07T19:34:05.664+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.664+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:05.664+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.664+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:05.666+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.666+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 185 ==== pg_notify2(1.2s0 1.2s0 (query:28 sent:28 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954225517s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e28/28) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e83c9ea00 con 0x558e818d7c00 2019-12-07T19:34:05.667+0000 7fe2e7d3c700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 1(2) notify: (query:28 sent:28 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954225517s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.667+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 1(2) notify: (query:28 sent:28 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954225517s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 1(2) notify: (query:28 sent:28 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954225517s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 1(2) notify: (query:28 sent:28 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954225517s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.667+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 186 ==== pg_query2(1.9s3 1.9s3 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e84242a00 con 0x558e818d7c00 2019-12-07T19:34:05.667+0000 7fe2e7d3c700 15 osd.0 28 enqueue_peering_evt 1.9s3 epoch_sent: 28 epoch_requested: 28 MQuery 1.9s3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.667+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.9s3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 1(2) notify: (query:28 sent:28 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954225517s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] got osd.1(2) 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954225517s) 2019-12-07T19:34:05.667+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 187 ==== pg_query2(1.es4 1.es4 query(info 0'0 epoch_sent 28) e28/28) v1 ==== 158+0+0 (unknown 0 0 0) 0x558e84242a00 con 0x558e818d7c00 2019-12-07T19:34:05.667+0000 7fe2e7d3c700 15 osd.0 28 enqueue_peering_evt 1.es4 epoch_sent: 28 epoch_requested: 28 MQuery 1.es4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.667+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.es4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] reg_next_scrub pg 1.2s0 register next scrub, scrub time 2019-12-07T19:34:57.527560+0000, must = 0 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: Adding osd: 1 peer features: 3f01cfb8ffadffff 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: Common peer features: 3f01cfb8ffadffff 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: Common acting features: 3f01cfb8ffadffff 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: Common upacting features: 3f01cfb8ffadffff 2019-12-07T19:34:05.667+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] exit Started/Primary/Peering/GetInfo 2019-12-07T19:34:05.667780+0000 11 0.000302 2019-12-07T19:34:05.667+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] enter Started/Primary/Peering/GetLog 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.0(0) 1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.1(2) 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954225517s) 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.3(5) 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956606539s) 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.4(1) 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956942681s) 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.5(3) 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 2019-12-07T19:34:05.667+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.5(4) 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=28) 2019-12-07T19:34:05.667+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.es4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.667+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.667+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.es4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e81a98000 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting all_info osd.7(4) 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 2019-12-07T19:34:05.667+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.949289263s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.es4 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] For position 0: selecting up[i]: 0(0) For position 1: selecting up[i]: 4(1) For position 2: selecting up[i]: 1(2) For position 3: selecting up[i]: 5(3) For position 4: selecting up[i]: 7(4) For position 5: selecting up[i]: 3(5) 2019-12-07T19:34:05.667+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.938428616s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.667+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:05.667+0000 7fe2c4a60700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.es0 1.es0 (query:28 sent:28 1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.938428616s) 4->0 ([18,27] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),6(5) intervals=([26,27] acting 0(4),1(0),3(1),4(2),5(3)))) e28/28) v1 -- 0x558e84707800 con 0x558e818d7c00 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] choose_async_recovery_ec candidates by cost are: 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] choose_async_recovery_ec result want=[0,4,1,5,7,3] async_recovery= 2019-12-07T19:34:05.667+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] acting_recovery_backfill is 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] choose_acting want=[0,4,1,5,7,3] backfill_targets= async_recovery_targets= 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: leaving GetLog 2019-12-07T19:34:05.667+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.667+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.9s3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.667+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:05.667+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] exit Started/Primary/Peering/GetLog 2019-12-07T19:34:05.667950+0000 0 0.000000 2019-12-07T19:34:05.667+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MQuery 1.9s3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28)) prio 255 cost 10 e28) pg 0x558e8192b000 2019-12-07T19:34:05.667+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] enter Started/Primary/Peering/GetMissing 2019-12-07T19:34:05.667+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.943013621s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MQuery 1.9s3 from 1(0) query_epoch 28 query: query(info 0'0 epoch_sent 28) 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: osd.1(2) has no missing, identical log 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: osd.3(5) has no missing, identical log 2019-12-07T19:34:05.667+0000 7fe2c7265700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.937977593s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] sending info 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: osd.4(1) has no missing, identical log 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: osd.5(3) has no missing, identical log 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: osd.7(4) has no missing, identical log 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: still need up_thru update before going active 2019-12-07T19:34:05.667+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] exit Started/Primary/Peering/GetMissing 2019-12-07T19:34:05.668014+0000 0 0.000000 2019-12-07T19:34:05.667+0000 7fe2c6a64700 5 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] enter Started/Primary/Peering/WaitUpThru 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e83e73200 seq 30 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e73200 cost 4032 (1 ios * 4000 + 32 bytes) 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e73200 onodes shared_blobs 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e73200 allocated 0x[] released 0x[] 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73200 prepare 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e73200 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73200 io_done 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 28 queue_want_up_thru want 28 <= queued 28, currently 27 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.667+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:05.667+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:05.667+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.667+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 23 < min_flush_size 524288 2019-12-07T19:34:05.667+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:05.667+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 46 < min_flush_size 524288 2019-12-07T19:34:05.667+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.667+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xdcf8c~2e to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.667+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xdcf8c 0xdcf8c~2e to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.667+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:05.667+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xacf8c 2019-12-07T19:34:05.667+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xf8c 2019-12-07T19:34:05.667+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:05.667+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xfba and padding block with 0x46 2019-12-07T19:34:05.667+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:05.667+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:28 sent:28 1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.937977593s) 3->0 ([18,27] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([26,27] acting 0(3),1(0),4(1),5(2),7(4)))) e28/28) v1 -- 0x558e8404d100 con 0x558e818d7c00 2019-12-07T19:34:05.667+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:05.667+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xdcfba 2019-12-07T19:34:05.667+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:05.667+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:05.667+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 105 ==== pg_notify2(1.2s0 1.2s0 (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.942675381s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e28/28) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e83c48a00 con 0x558e818d7000 2019-12-07T19:34:05.667+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:05.667+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:05.667+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000127016s (0.000000092s flush + 0.000126924s kv commit) 2019-12-07T19:34:05.667+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:05.667+0000 7fe2e7d3c700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 3(5) notify: (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.942675381s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.667+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 3(5) notify: (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.942675381s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 3(5) notify: (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.942675381s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 3(5) notify: (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.942675381s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 3(5) notify: (query:28 sent:28 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.942675381s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.667+0000 7fe2c6a64700 7 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: handle_pg_notify from osd.3(5) 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] got dup osd.3(5) info 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.942675381s), identical to ours 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:05.667+0000 7fe2c6a64700 10 osd.0 28 queue_want_up_thru want 28 <= queued 28, currently 27 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:05.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e73200> 2019-12-07T19:34:05.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:05.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73200 kv_submitted 2019-12-07T19:34:05.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e73200 2019-12-07T19:34:05.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73200 finishing 2019-12-07T19:34:05.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e73200 onodes 2019-12-07T19:34:05.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e73200 done 2019-12-07T19:34:05.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:05.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e73200 [] 2019-12-07T19:34:05.667+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:05.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:05.667+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.667+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.668+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 188 ==== pg_notify2(1.2s0 1.2s0 (query:28 sent:28 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.942812290s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e28/28) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e83c48a00 con 0x558e818d7c00 2019-12-07T19:34:05.668+0000 7fe2e7d3c700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 1(2) notify: (query:28 sent:28 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.942812290s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.668+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 1(2) notify: (query:28 sent:28 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.942812290s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) 2019-12-07T19:34:05.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:05.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 1(2) notify: (query:28 sent:28 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.942812290s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) queued 2019-12-07T19:34:05.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28)> waiting <> waiting_peering {} 2019-12-07T19:34:05.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 1(2) notify: (query:28 sent:28 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.942812290s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:05.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 MNotifyRec 1.2s0 from 1(2) notify: (query:28 sent:28 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.942812290s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:05.668+0000 7fe2caa6c700 7 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: handle_pg_notify from osd.1(2) 2019-12-07T19:34:05.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] got dup osd.1(2) info 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.942812290s), identical to ours 2019-12-07T19:34:05.668+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:05.668+0000 7fe2caa6c700 10 osd.0 28 queue_want_up_thru want 28 <= queued 28, currently 27 2019-12-07T19:34:05.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:05.716+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.767+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.817+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.867+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.917+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:05.967+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.017+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.067+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.117+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.168+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.217+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.268+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.318+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.342+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.2 172.21.15.131:0/37077 8 ==== osd_ping(ping e28 up_from 13 ping_stamp 2019-12-07T19:34:06.343400+0000/29.815119378s send_stamp 29.815119378s delta_ub 0.017113223s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e8182c000 2019-12-07T19:34:06.342+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.2 172.21.15.131:0/37077 8 ==== osd_ping(ping e28 up_from 13 ping_stamp 2019-12-07T19:34:06.343400+0000/29.815119378s send_stamp 29.815119378s delta_ub 0.017113223s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84197000 con 0x558e8182d400 2019-12-07T19:34:06.342+0000 7fe2e6d3a700 20 osd.0 28 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016879515s,0.017113223s]) 2019-12-07T19:34:06.342+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e28 up_from 13 ping_stamp 2019-12-07T19:34:06.343400+0000/29.815119378s send_stamp 29.798239863s delta_ub 18446744073.692672101s) v5 -- 0x558e818efc00 con 0x558e8182c000 2019-12-07T19:34:06.342+0000 7fe2e7d3c700 20 osd.0 28 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016836991s,0.017113223s]) 2019-12-07T19:34:06.342+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e28 up_from 13 ping_stamp 2019-12-07T19:34:06.343400+0000/29.815119378s send_stamp 29.798282387s delta_ub 18446744073.692714625s) v5 -- 0x558e81965000 con 0x558e8182d400 2019-12-07T19:34:06.368+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.418+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.468+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.518+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 10 osd.0 28 tick_without_osd_lock 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fa7d000/0x40000000/0x1680000000, data 0x57514d/0x581000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 25 osd.0 28 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254252+0000 last_rx_front 2019-12-07T19:34:02.254310+0000 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 25 osd.0 28 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254273+0000 last_rx_front 2019-12-07T19:34:02.254232+0000 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 25 osd.0 28 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254209+0000 last_rx_front 2019-12-07T19:34:02.254194+0000 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 25 osd.0 28 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254409+0000 last_rx_front 2019-12-07T19:34:02.254290+0000 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 25 osd.0 28 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254354+0000 last_rx_front 2019-12-07T19:34:02.254301+0000 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 25 osd.0 28 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254368+0000 last_rx_front 2019-12-07T19:34:02.254333+0000 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 20 osd.0 28 can_inc_scrubs == true 0 local + 0 remote < max 1 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 20 osd.0 28 scrub_time_permit should run between 0 - 24 now 19 = yes 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 20 osd.0 28 scrub_load_below_threshold loadavg per cpu 0.065 < max 5 = yes 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 20 osd.0 28 sched_scrub load_is_low=1 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 10 osd.0 28 sched_scrub 1.2s0 scheduled at 2019-12-07T19:34:57.527560+0000 > 2019-12-07T19:34:06.546295+0000 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 20 osd.0 28 sched_scrub done 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 10 osd.0 28 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 20 osd.0 28 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:06.545+0000 7fe2e2ca2700 10 osd.0 28 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:06.568+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258966 2019-12-07T19:34:06.618+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 1 osrs, 2 txcs 2019-12-07T19:34:06.618+0000 7fe2dc28f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock osr 0x558e8107a500 2 ios pending 2019-12-07T19:34:06.618+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 18 0x2000~1000 2019-12-07T19:34:06.618+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x2000~1000 crc f8c23e13 2019-12-07T19:34:06.618+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 18 0x1b000~1000 2019-12-07T19:34:06.618+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x1b000~1000 crc cdd76f83 2019-12-07T19:34:06.618+0000 7fe2dc28f700 5 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 562375 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.618+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish osr 0x558e8107a500 2019-12-07T19:34:06.618+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish dequeueing 2019-12-07T19:34:06.636+0000 7fe2e4535700 10 osd.0 28 tick 2019-12-07T19:34:06.636+0000 7fe2e4535700 10 osd.0 28 do_waiters -- start 2019-12-07T19:34:06.636+0000 7fe2e4535700 10 osd.0 28 do_waiters -- finish 2019-12-07T19:34:06.636+0000 7fe2e4535700 20 osd.0 28 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:06.655+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 106 ==== pg_notify2(1.2s0 1.2s0 (query:29 sent:29 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.955112918s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e29/29) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e81a5a300 con 0x558e818fe400 2019-12-07T19:34:06.655+0000 7fe2e6d3a700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 4(1) notify: (query:29 sent:29 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.955112918s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.655+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 4(1) notify: (query:29 sent:29 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.955112918s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 4(1) notify: (query:29 sent:29 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.955112918s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.655+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 4(1) notify: (query:29 sent:29 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.955112918s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.655+0000 7fe2c6a64700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 29, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 4(1) notify: (query:29 sent:29 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.955112918s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.655+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 108 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e81a5a300 con 0x558e818fe400 2019-12-07T19:34:06.655+0000 7fe2e6d3a700 15 osd.0 28 enqueue_peering_evt 1.bs1 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.104938842s int 16s) 2019-12-07T19:34:06.655+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.104938842s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.104938842s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.655+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.655+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.104938842s int 16s)) prio 255 cost 10 e29) pg 0x558e83c78000 2019-12-07T19:34:06.655+0000 7fe2ca26b700 20 osd.0 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 29, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.104938842s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.655+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.658+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 107 ==== osd_map(29..29 src has 1..29) v4 ==== 681+0+0 (unknown 0 0 0) 0x558e84409d40 con 0x558e818fe400 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 osd.0 28 OSD::ms_dispatch: osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- start 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- finish 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 osd.0 28 _dispatch 0x558e84409d40 osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.658+0000 7fe2d4a80700 3 osd.0 28 handle_osd_map epochs [29,29], i have 28, src has [1,29] 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 osd.0 28 handle_osd_map got inc map for epoch 29 2019-12-07T19:34:06.658+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3058e826:::osdmap.28:0# 0x0~0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1454 size 0x1454 (5204) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f941c0 blob([0x581000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f947e0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83af8e00 blob([!~1000,0x582000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558e83edf810 sbid 0x0)) need 0x1000~454 cache has 0x[1000~454] 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3058e826:::osdmap.28:0# 0x0~1454 = 5204 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 osd.0 28 add_map_bl 28 5204 bytes 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 osd.0 28 got_full_map 29, nothing requested 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 osd.0 28 handle_osd_map pg_num_history pg_num_history(e29 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps purged_snaps {29={}} 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 osd.0 28 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e29 [1,29] lci=[13,29]) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e83e72000 seq 21 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c34b972f:::inc_osdmap.29:0# key 0x7f7fffffffffffffffc34b97'/!inc_osdmap.29!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c34b972f:::inc_osdmap.29:0# 0x0~27d 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1147 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c34b972f:::inc_osdmap.29:0# 0x0~27d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~27d 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xd83 on front/back, now 0x0~1000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72000 1 blobs 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x583000~1000/1000,1000,0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x583000~1000] 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84549b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84549c00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84549b90 blob([0x583000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84549c00 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.blob(0x558e84549b90) get_ref 0x0~27d Blob(0x558e84549b90 blob([0x583000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84549c00 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.blob(0x558e84549b90) get_ref init 0x1000, 1000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~27d: 0x0~27d Blob(0x558e84549b90 blob([0x583000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x27d) SharedBlob(0x558e84549c00 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e84549c18 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x27d 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191bc88) dirty_range mark inline shard dirty 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c34b972f:::inc_osdmap.29:0# 0x0~27d = 0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3bd8e826:::osdmap.29:0# key 0x7f7fffffffffffffff3bd8e8'&!osdmap.29!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3bd8e826:::osdmap.29:0# 0x0~1438 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1148 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3bd8e826:::osdmap.29:0# 0x0~1438 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~438 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbc8 on front/back, now 0x0~1000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72000 2 blobs 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x584000~2000/1000,2000,0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x584000~2000] 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7ba40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b9d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7ba40 blob([0x584000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b9d0 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.blob(0x558e81a7ba40) get_ref 0x0~1000 Blob(0x558e81a7ba40 blob([0x584000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b9d0 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.blob(0x558e81a7ba40) get_ref init 0x1000, 1000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e81a7ba40 blob([0x584000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e81a7b9d0 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81a7b9e8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423c770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423c7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423c770 blob([!~1000,0x585000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423c7e0 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.blob(0x558e8423c770) get_ref 0x1000~438 Blob(0x558e8423c770 blob([!~1000,0x585000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e8423c7e0 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.blob(0x558e8423c770) get_ref init 0x2000, 1000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~438: 0x1000~438 Blob(0x558e8423c770 blob([!~1000,0x585000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558e8423c7e0 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e8423c7f8 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1438 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83c73788) dirty_range mark inline shard dirty 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3bd8e826:::osdmap.29:0# 0x0~1438 = 0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72000 0 blobs 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72000 0 blobs 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72000 cost 23294 (4 ios * 4000 + 7294 bytes) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72000 onodes 0x558e80251b80,0x558e817b9180,0x558e8191bb80,0x558e83c73680 shared_blobs 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191bc88) update #-1:c34b972f:::inc_osdmap.29:0# 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8191bc88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c34b972f:::inc_osdmap.29:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83c73788) update #-1:3bd8e826:::osdmap.29:0# 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83c73788) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3bd8e826:::osdmap.29:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72000 allocated 0x[583000~3000] released 0x[] 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 freelist allocate 0x583000~3000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 20 freelist _xor first_key 0x580000 last_key 0x580000 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 prepare 2019-12-07T19:34:06.658+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83e72000 2019-12-07T19:34:06.658+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 32 ==== pg_notify2(1.2s0 1.2s0 (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e29/29) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e84704000 con 0x558e818fe000 2019-12-07T19:34:06.658+0000 7fe2e753b700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.658+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.658+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 51 ==== pg_notify2(1.2s0 1.2s0 (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e29/29) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e84704000 con 0x558e818d6800 2019-12-07T19:34:06.658+0000 7fe2e753b700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.658+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.658+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 189 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d100 con 0x558e818d7c00 2019-12-07T19:34:06.658+0000 7fe2e7d3c700 15 osd.0 28 enqueue_peering_evt 1.8s5 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.113761808s int 16s) 2019-12-07T19:34:06.658+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 53 ==== pg_notify2(1.2s0 1.2s0 (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e29/29) v1 ==== 1237+0+0 (unknown 0 0 0) 0x558e84704000 con 0x558e818d6800 2019-12-07T19:34:06.658+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.113761808s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.658+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 190 ==== pg_info2(1.9s3 1.9s3 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d100 con 0x558e818d7c00 2019-12-07T19:34:06.658+0000 7fe2e753b700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.658+0000 7fe2e7d3c700 15 osd.0 28 enqueue_peering_evt 1.9s3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114035089s int 16s) 2019-12-07T19:34:06.658+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.658+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114035089s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.658+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 106 ==== pg_notify2(1.2s0 1.2s0 (query:29 sent:29 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953485642s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e29/29) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e8404d100 con 0x558e818d7000 2019-12-07T19:34:06.659+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.659+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114035089s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.659+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.659+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114035089s int 16s)) prio 255 cost 10 e29) pg 0x558e8192b000 2019-12-07T19:34:06.659+0000 7fe2cb26d700 20 osd.0 op_wq _add_slot_waiter 1.9s3 peering, item epoch is 29, will wait on OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114035089s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.659+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.659+0000 7fe2e7d3c700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 3(5) notify: (query:29 sent:29 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953485642s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.659+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 3(5) notify: (query:29 sent:29 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953485642s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.659+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 107 ==== pg_info2(1.fs1 1.fs1 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d100 con 0x558e818d7000 2019-12-07T19:34:06.659+0000 7fe2e7d3c700 15 osd.0 28 enqueue_peering_evt 1.fs1 epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114604348s int 16s) 2019-12-07T19:34:06.659+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114604348s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.659+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 108 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d100 con 0x558e818d7000 2019-12-07T19:34:06.659+0000 7fe2e7d3c700 15 osd.0 28 enqueue_peering_evt 1.1s1 epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115024122s int 16s) 2019-12-07T19:34:06.659+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115024122s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {29=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)>} 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {29=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)>} 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 29, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {29=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)>} 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {29=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)>} 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 29, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {29=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)>} 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {29=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)>} 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 29, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {29=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)>} 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 3(5) notify: (query:29 sent:29 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953485642s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {29=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)>} 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 3(5) notify: (query:29 sent:29 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953485642s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 29, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 3(5) notify: (query:29 sent:29 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953485642s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.659+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.659+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.659+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115024122s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.659+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.659+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115024122s int 16s)) prio 255 cost 10 e29) pg 0x558e819e4000 2019-12-07T19:34:06.659+0000 7fe2c7265700 20 osd.0 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 29, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115024122s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.659+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.659+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 109 ==== pg_info2(1.3s5 1.3s5 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e81a5a300 con 0x558e818fe400 2019-12-07T19:34:06.659+0000 7fe2e6d3a700 15 osd.0 28 enqueue_peering_evt 1.3s5 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.105685108s int 16s) 2019-12-07T19:34:06.659+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.105685108s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.659+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 19 ==== osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8 ==== 234+0+0 (crc 0 0 0) 0x558e83ca7600 con 0x558e83d68000 2019-12-07T19:34:06.659+0000 7fe2e6d3a700 15 osd.0 28 enqueue_op 0x558e84176b00 prio 63 cost 0 latency 0.000016 epoch 29 osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8 2019-12-07T19:34:06.659+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8) prio 63 cost 0 e29) 2019-12-07T19:34:06.659+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.659+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.105685108s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.659+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.659+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.105685108s int 16s)) prio 255 cost 10 e29) pg 0x558e81231000 2019-12-07T19:34:06.659+0000 7fe2c6263700 20 osd.0 op_wq _add_slot_waiter 1.3s5 peering, item epoch is 29, will wait on OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.105685108s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.659+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.659+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {29=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 3(5) notify: (query:29 sent:29 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953485642s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)>} 2019-12-07T19:34:06.659+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8) prio 63 cost 0 e29) queued 2019-12-07T19:34:06.659+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {29=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 3(5) notify: (query:29 sent:29 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953485642s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)>} 2019-12-07T19:34:06.659+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8) prio 63 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.659+0000 7fe2c6a64700 10 osd.0 28 dequeue_op 0x558e84176b00 prio 63 cost 0 latency 0.000153 osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8 pg pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] 2019-12-07T19:34:06.659+0000 7fe2c6a64700 10 osd.0 28 maybe_share_map con 0x558e83d68000 172.21.15.131:0/224485236 map epoch 27 -> 29 (as per caller) 2019-12-07T19:34:06.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.659+0000 7fe2c6a64700 20 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] do_request min 29, queue on waiting_for_map client.4540 2019-12-07T19:34:06.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114604348s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114604348s int 16s)) prio 255 cost 10 e29) pg 0x558e81a9b000 2019-12-07T19:34:06.659+0000 7fe2c425f700 20 osd.0 op_wq _add_slot_waiter 1.fs1 peering, item epoch is 29, will wait on OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114604348s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.659+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- mon_subscribe({osdmap=29}) v3 -- 0x558e84053c00 con 0x558e80f39000 2019-12-07T19:34:06.659+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 26 ==== osd_map(29..29 src has 1..29) v4 ==== 681+0+0 (crc 0 0 0) 0x558e8123b440 con 0x558e80f39000 2019-12-07T19:34:06.659+0000 7fe2d959f700 20 osd.0 28 OSD::ms_dispatch: osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.659+0000 7fe2d959f700 10 osd.0 28 do_waiters -- start 2019-12-07T19:34:06.659+0000 7fe2d959f700 10 osd.0 28 do_waiters -- finish 2019-12-07T19:34:06.659+0000 7fe2c6a64700 10 osd.0 28 dequeue_op 0x558e84176b00 finish 2019-12-07T19:34:06.659+0000 7fe2d959f700 20 osd.0 28 _dispatch 0x558e8123b440 osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.659+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.659+0000 7fe2d959f700 3 osd.0 28 handle_osd_map epochs [29,29], i have 29, src has [1,29] 2019-12-07T19:34:06.659+0000 7fe2d959f700 10 osd.0 28 no new maps here, dropping 2019-12-07T19:34:06.659+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 aio_wait 2019-12-07T19:34:06.659+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72000 2019-12-07T19:34:06.659+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 io_done 2019-12-07T19:34:06.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:06.659+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:06.660+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:06.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:06.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:06.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8191bb80 had 1 2019-12-07T19:34:06.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c73680 had 1 2019-12-07T19:34:06.660+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:06.660+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3859939328 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:06.660+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998559) bluestore 86 GiB free (0.95994), bluefs_ratio 0.0399475 2019-12-07T19:34:06.660+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8836 < min_flush_size 524288 2019-12-07T19:34:06.660+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8836 < min_flush_size 524288 2019-12-07T19:34:06.660+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:06.660+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xdcfba~2284 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:06.660+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xdcfba 0xdcfba~2284 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:06.660+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:06.660+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~b0000 x_off 0xacfba 2019-12-07T19:34:06.660+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xfba 2019-12-07T19:34:06.660+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.660+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x23e and padding block with 0xdc2 2019-12-07T19:34:06.660+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 192 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d100 con 0x558e818d7c00 2019-12-07T19:34:06.660+0000 7fe2e7d3c700 15 osd.0 28 enqueue_peering_evt 1.es4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114825888s int 16s) 2019-12-07T19:34:06.660+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114825888s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.660+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 193 ==== pg_info2(1.as3 1.as3 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d100 con 0x558e818d7c00 2019-12-07T19:34:06.660+0000 7fe2e7d3c700 15 osd.0 28 enqueue_peering_evt 1.as3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115034256s int 16s) 2019-12-07T19:34:06.660+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115034256s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.660+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 194 ==== pg_info2(1.7s3 1.7s3 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d100 con 0x558e818d7c00 2019-12-07T19:34:06.660+0000 7fe2e7d3c700 15 osd.0 28 enqueue_peering_evt 1.7s3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.116162602s int 16s) 2019-12-07T19:34:06.660+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.116162602s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.660+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 195 ==== pg_notify2(1.2s0 1.2s0 (query:29 sent:29 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950531088s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) e29/29) v1 ==== 1366+0+0 (unknown 0 0 0) 0x558e8404d100 con 0x558e818d7c00 2019-12-07T19:34:06.660+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.660+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115034256s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.660+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.660+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115034256s int 16s)) prio 255 cost 10 e29) pg 0x558e8193b000 2019-12-07T19:34:06.660+0000 7fe2caa6c700 20 osd.0 op_wq _add_slot_waiter 1.as3 peering, item epoch is 29, will wait on OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115034256s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.660+0000 7fe2e7d3c700 15 osd.0 28 enqueue_peering_evt 1.2s0 epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 1(2) notify: (query:29 sent:29 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950531088s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.660+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.660+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 1(2) notify: (query:29 sent:29 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950531088s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.660+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {29=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 3(5) notify: (query:29 sent:29 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953485642s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)>} 2019-12-07T19:34:06.660+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 1(2) notify: (query:29 sent:29 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950531088s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.660+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {29=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 3(5) notify: (query:29 sent:29 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953485642s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)>} 2019-12-07T19:34:06.660+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 1(2) notify: (query:29 sent:29 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950531088s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.660+0000 7fe2c6a64700 20 osd.0 op_wq _add_slot_waiter 1.2s0 peering, item epoch is 29, will wait on OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 1(2) notify: (query:29 sent:29 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950531088s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) 2019-12-07T19:34:06.660+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.660+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.660+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114825888s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.660+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.660+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114825888s int 16s)) prio 255 cost 10 e29) pg 0x558e81a98000 2019-12-07T19:34:06.660+0000 7fe2c8a68700 20 osd.0 op_wq _add_slot_waiter 1.es4 peering, item epoch is 29, will wait on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114825888s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.660+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.660+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.660+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.116162602s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.660+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.660+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.116162602s int 16s)) prio 255 cost 10 e29) pg 0x558e8122e000 2019-12-07T19:34:06.660+0000 7fe2c8267700 20 osd.0 op_wq _add_slot_waiter 1.7s3 peering, item epoch is 29, will wait on OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.116162602s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.660+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.660+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 38 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e84704000 con 0x558e818d6000 2019-12-07T19:34:06.660+0000 7fe2e753b700 15 osd.0 28 enqueue_peering_evt 1.5s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.131453992s int 16s) 2019-12-07T19:34:06.660+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.131453992s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.660+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.660+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.131453992s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.660+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.660+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.131453992s int 16s)) prio 255 cost 10 e29) pg 0x558e83d07000 2019-12-07T19:34:06.660+0000 7fe2c9269700 20 osd.0 op_wq _add_slot_waiter 1.5s4 peering, item epoch is 29, will wait on OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.131453992s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.660+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:06.661+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 33 ==== osd_map(29..29 src has 1..29) v4 ==== 681+0+0 (unknown 0 0 0) 0x558e844098c0 con 0x558e818fe000 2019-12-07T19:34:06.661+0000 7fe2d4a80700 20 osd.0 28 OSD::ms_dispatch: osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- start 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- finish 2019-12-07T19:34:06.661+0000 7fe2d4a80700 20 osd.0 28 _dispatch 0x558e844098c0 osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.661+0000 7fe2d4a80700 3 osd.0 28 handle_osd_map epochs [29,29], i have 29, src has [1,29] 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 no new maps here, dropping 2019-12-07T19:34:06.661+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 39 ==== osd_map(29..29 src has 1..29) v4 ==== 681+0+0 (unknown 0 0 0) 0x558e8419eb40 con 0x558e818d6000 2019-12-07T19:34:06.661+0000 7fe2d4a80700 20 osd.0 28 OSD::ms_dispatch: osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- start 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- finish 2019-12-07T19:34:06.661+0000 7fe2d4a80700 20 osd.0 28 _dispatch 0x558e8419eb40 osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.661+0000 7fe2d4a80700 3 osd.0 28 handle_osd_map epochs [29,29], i have 29, src has [1,29] 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 no new maps here, dropping 2019-12-07T19:34:06.661+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 52 ==== osd_map(29..29 src has 1..29) v4 ==== 681+0+0 (unknown 0 0 0) 0x558e84409b00 con 0x558e818d6800 2019-12-07T19:34:06.661+0000 7fe2d4a80700 20 osd.0 28 OSD::ms_dispatch: osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- start 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- finish 2019-12-07T19:34:06.661+0000 7fe2d4a80700 20 osd.0 28 _dispatch 0x558e84409b00 osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.661+0000 7fe2d4a80700 3 osd.0 28 handle_osd_map epochs [29,29], i have 29, src has [1,29] 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 no new maps here, dropping 2019-12-07T19:34:06.661+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 191 ==== osd_map(29..29 src has 1..29) v4 ==== 681+0+0 (unknown 0 0 0) 0x558e841a06c0 con 0x558e818d7c00 2019-12-07T19:34:06.661+0000 7fe2d4a80700 20 osd.0 28 OSD::ms_dispatch: osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- start 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- finish 2019-12-07T19:34:06.661+0000 7fe2d4a80700 20 osd.0 28 _dispatch 0x558e841a06c0 osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.661+0000 7fe2d4a80700 3 osd.0 28 handle_osd_map epochs [29,29], i have 29, src has [1,29] 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 no new maps here, dropping 2019-12-07T19:34:06.661+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 109 ==== osd_map(29..29 src has 1..29) v4 ==== 681+0+0 (unknown 0 0 0) 0x558e83c66fc0 con 0x558e818d7000 2019-12-07T19:34:06.661+0000 7fe2d4a80700 20 osd.0 28 OSD::ms_dispatch: osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- start 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 do_waiters -- finish 2019-12-07T19:34:06.661+0000 7fe2d4a80700 20 osd.0 28 _dispatch 0x558e83c66fc0 osd_map(29..29 src has 1..29) v4 2019-12-07T19:34:06.661+0000 7fe2d4a80700 3 osd.0 28 handle_osd_map epochs [29,29], i have 29, src has [1,29] 2019-12-07T19:34:06.661+0000 7fe2d4a80700 10 osd.0 28 no new maps here, dropping 2019-12-07T19:34:06.662+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.662+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.113761808s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.662+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.662+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.113761808s int 16s)) prio 255 cost 10 e29) pg 0x558e841a4000 2019-12-07T19:34:06.662+0000 7fe2cba6e700 20 osd.0 op_wq _add_slot_waiter 1.8s5 peering, item epoch is 29, will wait on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.113761808s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.662+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xdf23e 2019-12-07T19:34:06.662+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.662+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.662+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.662+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.662+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 1 in 0.003103630s (0.000501976s flush + 0.002601654s kv commit) 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.662+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e72000> 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <0x558e83c6e140> 2019-12-07T19:34:06.662+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 kv_submitted 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72000 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e83e72000 osr 0x558e8107a500 2019-12-07T19:34:06.662+0000 7fe2d7285700 10 osd.0 28 _committed_osd_maps 29..29 2019-12-07T19:34:06.662+0000 7fe2d7285700 10 osd.0 28 advance to epoch 29 (<= last 29 <= newest_map 29) 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e846eb040) prepare_write seq 19 0x1b000~1000 crc d708c37e 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e846eb040) prepare_write seq 19 0x2000~1000 crc 43f92a2e 2019-12-07T19:34:06.662+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0600 deferred_cleanup 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0600 onodes 0x558e80251b80,0x558e817b9180,0x558e8191b680,0x558e83b08000 2019-12-07T19:34:06.662+0000 7fe2d7285700 7 osd.0 29 consume_map version 29 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d0d38 in 0x558e80219340) _finish_write discard buffer(0x558e83d11b90 space 0x558e811d0d38 0x1000~1000 writing nocache) 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 identify_splits_and_merges 1.8s5 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd20f8 in 0x558e80219340) _finish_write discard buffer(0x558e83d10090 space 0x558e83dd20f8 0x0~1000 writing nocache) 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 identify_splits_and_merges 1.9s3 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2168 in 0x558e80219340) _finish_write discard buffer(0x558e83d105a0 space 0x558e83dd2168 0x0~1000 writing nocache) 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 identify_splits_and_merges 1.1s1 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 identify_splits_and_merges 1.as3 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0600 done 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 identify_splits_and_merges 1.2s0 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fbb00 deferred_cleanup 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 identify_splits_and_merges 1.3s5 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.662+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0600 [] 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 identify_splits_and_merges 1.bs1 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.662+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 identify_splits_and_merges 1.bs3 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 identify_splits_and_merges 1.4s1 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 identify_splits_and_merges 1.5s4 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 identify_splits_and_merges 1.ds5 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 identify_splits_and_merges 1.es4 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.662+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 deferred_cleanup 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 identify_splits_and_merges 1.7s3 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fbb00 onodes 0x558e80251b80,0x558e817b9180,0x558e83f28c80,0x558e83f28f00 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 identify_splits_and_merges 1.fs1 e28 to e29 pg_nums {18=16} 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 29 prune_pg_created 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edf828 in 0x558e80219340) _finish_write discard buffer(0x558e84032870 space 0x558e83edf828 0x1000~1000 writing nocache) 2019-12-07T19:34:06.662+0000 7fe2d7285700 10 osd.0:0.consume_map 29 (was 28) 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f940f8 in 0x558e80219340) _finish_write discard buffer(0x558e84032990 space 0x558e83f940f8 0x0~1000 writing nocache) 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f947f8 in 0x558e80219340) _finish_write discard buffer(0x558e84032900 space 0x558e83f947f8 0x0~1000 writing nocache) 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 pending_peering first epoch 29 <= 29, requeueing 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {29=} 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fbb00 done 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72000 deferred_queued 2019-12-07T19:34:06.662+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fbb00 [] 2019-12-07T19:34:06.662+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.662+0000 7fe2d7285700 10 osd.0:1.consume_map 29 (was 28) 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 pending_peering first epoch 29 <= 29, requeueing 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.9s3 to_process <> waiting <> waiting_peering {29=} 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:06.662+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 pending_peering first epoch 29 <= 29, requeueing 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {29=} 2019-12-07T19:34:06.662+0000 7fe2d7285700 10 osd.0:2.consume_map 29 (was 28) 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 pending_peering first epoch 29 <= 29, requeueing 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.as3 to_process <> waiting <> waiting_peering {29=} 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:06.662+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 pending_peering first epoch 29 <= 29, requeueing 2019-12-07T19:34:06.662+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115024122s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.662+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.662+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115024122s int 16s)) prio 255 cost 10 e29) pg 0x558e819e4000 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.2s0 to_process <> waiting <> waiting_peering {29=0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 3(5) notify: (query:29 sent:29 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953485642s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29),OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 1(2) notify: (query:29 sent:29 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950531088s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)>} 2019-12-07T19:34:06.662+0000 7fe2d7285700 10 osd.0:3.consume_map 29 (was 28) 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 pending_peering first epoch 29 <= 29, requeueing 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.3s5 to_process <> waiting <> waiting_peering {29=} 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 pending_peering first epoch 29 <= 29, requeueing 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {29=} 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 2019-12-07T19:34:06.662+0000 7fe2d7285700 10 osd.0:4.consume_map 29 (was 28) 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:06.662+0000 7fe2d7285700 10 osd.0:5.consume_map 29 (was 28) 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 pending_peering first epoch 29 <= 29, requeueing 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:5._wake_pg_slot _wake_pg_slot 1.5s4 to_process <> waiting <> waiting_peering {29=} 2019-12-07T19:34:06.662+0000 7fe2cb26d700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947291363s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.662+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.1s1 28 -> 29 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:06.662+0000 7fe2d7285700 10 osd.0:6.consume_map 29 (was 28) 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 pending_peering first epoch 29 <= 29, requeueing 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {29=} 2019-12-07T19:34:06.662+0000 7fe2cb26d700 10 osd.0 pg_epoch: 28 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947291363s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] handle_advance_map [3,0,2,4,5,1]/[3,0,2,4,5,1] -- 3/3 2019-12-07T19:34:06.662+0000 7fe2d7285700 10 osd.0:7.consume_map 29 (was 28) 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 pending_peering first epoch 29 <= 29, requeueing 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.7s3 to_process <> waiting <> waiting_peering {29=} 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 pending_peering first epoch 29 <= 29, requeueing 2019-12-07T19:34:06.662+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947291363s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.fs1 to_process <> waiting <> waiting_peering {29=} 2019-12-07T19:34:06.662+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947291363s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.662+0000 7fe2d7285700 10 osd.0 29 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:06.662+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:34:06.662+0000 7fe2d7285700 15 osd.0 29 enqueue_peering_evt 1.8s5 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7fe2d7285700 15 osd.0 29 enqueue_peering_evt 1.9s3 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.662+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947291363s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7fe2d7285700 15 osd.0 29 enqueue_peering_evt 1.1s1 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947291363s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.662+0000 7fe2d7285700 15 osd.0 29 enqueue_peering_evt 1.as3 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7fe2d7285700 15 osd.0 29 enqueue_peering_evt 1.2s0 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947291363s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.662+0000 7fe2d7285700 15 osd.0 29 enqueue_peering_evt 1.3s5 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.662+0000 7fe2cb26d700 20 osd.0 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7fe2d7285700 15 osd.0 29 enqueue_peering_evt 1.bs1 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7fe2d7285700 15 osd.0 29 enqueue_peering_evt 1.bs3 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.662+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947291363s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7fe2d7285700 15 osd.0 29 enqueue_peering_evt 1.4s1 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.947291363s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.662+0000 7fe2d7285700 15 osd.0 29 enqueue_peering_evt 1.5s4 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7fe2d7285700 15 osd.0 29 enqueue_peering_evt 1.ds5 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7fe2d7285700 15 osd.0 29 enqueue_peering_evt 1.es4 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7fe2d7285700 15 osd.0 29 enqueue_peering_evt 1.7s3 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.662+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.662+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.662+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:06.662+0000 7fe2d7285700 15 osd.0 29 enqueue_peering_evt 1.fs1 epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.662+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115024122s int 16s) 2019-12-07T19:34:06.663+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) 2019-12-07T19:34:06.663+0000 7fe2d7285700 7 osd.0 29 activate_map version 29 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] state: got info from osd.3(0) 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114825888s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.115024122s int 16s) 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114825888s int 16s)) prio 255 cost 10 e29) pg 0x558e81a98000 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] peer_clock_delta_lb 0.002657868s -> ruub 46.112366254s 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.113761808s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.113761808s int 16s)) prio 255 cost 10 e29) pg 0x558e841a4000 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.112366254s 2019-12-07T19:34:06.663+0000 7fe2cb26d700 5 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.663726+0000 5 0.000100 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.938428616s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.663+0000 7fe2cb26d700 5 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0:0.update_pg_epoch 1.8s5 28 -> 29 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0:6.update_pg_epoch 1.es4 28 -> 29 2019-12-07T19:34:06.663+0000 7fe2cb26d700 5 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 pg_epoch: 28 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] handle_advance_map [1,4,5,2,3,0]/[1,4,5,2,3,0] -- 1/1 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 osd.0 pg_epoch: 28 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.938428616s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] handle_advance_map [1,3,4,5,0,2]/[1,3,4,5,0,2] -- 1/1 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown NOTIFY pruub 44.059111212s@ mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.938428616s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] activate - no missing, moving last_complete 27'7 -> 27'7 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.663+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.8s5_head options 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.938428616s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.663+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.938428616s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.938428616s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 29 send_incremental_map 28 -> 29 to 0x558e818d7000 v2:172.21.15.131:6812/37076 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.938428616s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.950381444s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.663+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c34b972f:::inc_osdmap.29:0# 0x0~0 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~27d size 0x27d (637) 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.938428616s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.938428616s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84549b90 blob([0x583000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x27d) SharedBlob(0x558e84549c00 sbid 0x0)) need 0x0~27d cache has 0x[0~27d] 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:c34b972f:::inc_osdmap.29:0# 0x0~27d = 637 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 29 add_map_inc_bl 29 637 bytes 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.113761808s int 16s) 2019-12-07T19:34:06.663+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- osd_map(29..29 src has 1..29) v4 -- 0x558e81a11d40 con 0x558e818d7000 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] state: got info from osd.1(0) 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.113761808s int 16s) 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114825888s int 16s) 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 29 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 28 -> 29 (shared) 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] peer_clock_delta_lb 0.001072657s -> ruub 46.112689151s 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] state: got info from osd.1(0) 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.112689151s 2019-12-07T19:34:06.663+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.1s0 1.1s0 (query:29 sent:29 1.1s1( v 27'7 (0'0,27'7] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) 1->0 ([18,27] all_participants=0(1),1(5),2(2),3(0),4(3),5(4),6(2) intervals=([26,27] acting 0(1),1(5),3(0),4(3),5(4)))) e29/29) v1 -- 0x558e81a5b800 con 0x558e818d7000 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.114825888s int 16s) 2019-12-07T19:34:06.663+0000 7fe2c7a66700 5 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.663851+0000 5 0.000095 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] peer_clock_delta_lb 0.001072657s -> ruub 46.113753231s 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e83d4d500 seq 23 2019-12-07T19:34:06.663+0000 7fe2c7a66700 5 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.113753231s 2019-12-07T19:34:06.663+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:06.663+0000 7fe2c7a66700 5 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:06.663+0000 7fe2c4a60700 5 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.663871+0000 5 0.000095 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown NOTIFY pruub 44.065109905s@ mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4d500 cost 5543 (1 ios * 4000 + 1543 bytes) 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4d500 onodes shared_blobs 2019-12-07T19:34:06.663+0000 7fe2c4a60700 5 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4d500 allocated 0x[] released 0x[] 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown pruub 44.065109905s@ mbc={} ps=[2~1]] activate - no missing, moving last_complete 27'23 -> 27'23 2019-12-07T19:34:06.663+0000 7fe2c4a60700 5 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown pruub 44.065109905s@ mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 prepare 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4d500 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 io_done 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown NOTIFY pruub 44.060997103s@ mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] activate - no missing, moving last_complete 21'4 -> 21'4 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114035089s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 29 send_incremental_map 28 -> 29 to 0x558e818d7c00 v2:172.21.15.131:6823/37078 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114035089s int 16s)) prio 255 cost 10 e29) pg 0x558e8192b000 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.663+0000 7fe2c7a66700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- osd_map(29..29 src has 1..29) v4 -- 0x558e83f2ab40 con 0x558e818d7c00 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 29 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 28 -> 29 (shared) 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.937977593s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.9s3 28 -> 29 2019-12-07T19:34:06.663+0000 7fe2c7a66700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.8s0 1.8s0 (query:29 sent:29 1.8s5( v 27'23 (0'0,27'23] local-lis/les=0/0 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) 5->0 ([18,27] all_participants=0(5),1(0),2(3),3(4),4(1),5(2),6(5) intervals=([26,27] acting 1(0),2(3),3(4),4(1),5(2)))) e29/29) v1 -- 0x558e83eba000 con 0x558e818d7c00 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 28 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.937977593s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e83c56c00 seq 24 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.937977593s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.663+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._biginfo' <- _biginfo 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.937977593s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.663+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:34:06.663+0000 7fe2c4a60700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.es0 1.es0 (query:29 sent:29 1.es4( v 21'4 (0'0,21'4] local-lis/les=26/27 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) 4->0 ([18,27] all_participants=0(4),1(0),2(5),3(1),4(2),5(3),6(5) intervals=([26,27] acting 0(4),1(0),3(1),4(2),5(3)))) e29/29) v1 -- 0x558e81a5aa00 con 0x558e818d7c00 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._epoch' <- _epoch 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.937977593s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56c00 cost 5543 (1 ios * 4000 + 1543 bytes) 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56c00 onodes shared_blobs 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56c00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.937977593s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e8417a600 seq 21 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 prepare 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.937977593s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56c00 2019-12-07T19:34:06.663+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 io_done 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.937977593s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.937977593s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558e841a4000 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417a600 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417a600 onodes shared_blobs 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417a600 allocated 0x[] released 0x[] 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown pruub 44.065109905s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.663+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 prepare 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417a600 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 io_done 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114035089s int 16s) 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114604348s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114604348s int 16s)) prio 255 cost 10 e29) pg 0x558e81a9b000 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] state: got info from osd.1(0) 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558e81a98000 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.114035089s int 16s) 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946712057s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] peer_clock_delta_lb 0.001072657s -> ruub 46.112962432s 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.fs1 28 -> 29 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.663+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.112962432s 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946712057s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] handle_advance_map [3,0,5,4,1,2]/[3,0,5,4,1,2] -- 3/3 2019-12-07T19:34:06.663+0000 7fe2cb26d700 5 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.664045+0000 5 0.000094 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946712057s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.663+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.663+0000 7fe2cb26d700 5 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946712057s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.663+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:34:06.663+0000 7fe2cb26d700 5 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946712057s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.060646370s@ mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946712057s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946712057s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] activate - no missing, moving last_complete 27'8 -> 27'8 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946712057s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946712057s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.114604348s int 16s) 2019-12-07T19:34:06.663+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.9s0 1.9s0 (query:29 sent:29 1.9s3( v 27'8 (0'0,27'8] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) 3->0 ([18,27] all_participants=0(3),1(0),3(5),4(1),5(2),6(5),7(4) intervals=([26,27] acting 0(3),1(0),4(1),5(2),7(4)))) e29/29) v1 -- 0x558e81229c00 con 0x558e818d7c00 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] state: got info from osd.3(0) 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83d4c900 seq 23 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.114604348s int 16s) 2019-12-07T19:34:06.663+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] peer_clock_delta_lb 0.002657868s -> ruub 46.111946480s 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.111946480s 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c900 cost 5543 (1 ios * 4000 + 1543 bytes) 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c900 onodes shared_blobs 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c900 allocated 0x[] released 0x[] 2019-12-07T19:34:06.663+0000 7fe2c425f700 5 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.664142+0000 5 0.000085 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 prepare 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c900 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 io_done 2019-12-07T19:34:06.663+0000 7fe2c425f700 5 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.663+0000 7fe2c425f700 5 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558e8192b000 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown NOTIFY pruub 44.059243341s@ mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] activate - no missing, moving last_complete 27'10 -> 27'10 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558e819e4000 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.663+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_notify2(1.fs0 1.fs0 (query:29 sent:29 1.fs1( v 27'10 (0'0,27'10] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) 1->0 ([18,27] all_participants=0(1),1(4),2(5),3(0),4(3),5(2),6(5) intervals=([26,27] acting 0(1),1(4),3(0),4(3),5(2)))) e29/29) v1 -- 0x558e846f8e00 con 0x558e818d7000 2019-12-07T19:34:06.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:06.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e83d4cc00 seq 29 2019-12-07T19:34:06.663+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1387 < min_flush_size 524288 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4cc00 cost 5543 (1 ios * 4000 + 1543 bytes) 2019-12-07T19:34:06.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4cc00 onodes shared_blobs 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4cc00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2774 < min_flush_size 524288 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 prepare 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4cc00 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 io_done 2019-12-07T19:34:06.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3522 < min_flush_size 524288 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4054 < min_flush_size 524288 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.116162602s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.116162602s int 16s)) prio 255 cost 10 e29) pg 0x558e8122e000 2019-12-07T19:34:06.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5441 < min_flush_size 524288 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 4(1) notify: (query:29 sent:29 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.955112918s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.663+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5464 < min_flush_size 524288 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 4(1) notify: (query:29 sent:29 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.955112918s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5464 < min_flush_size 524288 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xdf23e~1558 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948214015s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.7s3 28 -> 29 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xdf23e 0xdf23e~1558 to file(ino 16 size 0xe0000 mtime 2019-12-07T19:34:04.780083+0000 allocated e0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~b0000]) 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 bluefs _allocate len 0x796 from 0 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 bluefs _allocate len 0x796 from 1 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd210000 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd210000~10000/10000,10000,acd210000 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.663+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0xf0000 to include allocated 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.2s0 28 -> 29 2019-12-07T19:34:06.663+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 34 (was clean) 2019-12-07T19:34:06.663+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.663+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xaf23e 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 28 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948214015s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:06.663+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x23e 2019-12-07T19:34:06.663+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.663+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x796 and padding block with 0x86a 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 28 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:06.663+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0xf0000 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948214015s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: Peering advmap 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948214015s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.663+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] adjust_need_up_thru now 28, need_up_thru now false 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948214015s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948214015s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.663+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948214015s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948214015s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.948214015s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.116162602s int 16s) 2019-12-07T19:34:06.663+0000 7fe2caa6c700 7 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: handle ActMap primary 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] state: got info from osd.1(0) 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.116162602s int 16s) 2019-12-07T19:34:06.663+0000 7fe2caa6c700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] publish_stats_to_osd 28:69 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] peer_clock_delta_lb 0.001072657s -> ruub 46.115089945s 2019-12-07T19:34:06.663+0000 7fe2caa6c700 5 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] exit Started/Primary/Peering/WaitUpThru 2019-12-07T19:34:06.664388+0000 4 0.000349 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.115089945s 2019-12-07T19:34:06.663+0000 7fe2c425f700 5 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.664398+0000 5 0.000092 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] state: Leaving Peering 2019-12-07T19:34:06.663+0000 7fe2c425f700 5 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.663+0000 7fe2caa6c700 5 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 peering mbc={} ps=[2~1]] exit Started/Primary/Peering 2019-12-07T19:34:06.664408+0000 0 0.000000 2019-12-07T19:34:06.663+0000 7fe2c425f700 5 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown NOTIFY pruub 44.061193946s@ mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.663+0000 7fe2caa6c700 5 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] enter Started/Primary/Active 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'9 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] activate - no missing, moving last_complete 27'10 -> 27'10 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] state: In Active, about to call activate 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate - no missing, moving last_complete 27'12 -> 27'12 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] rollforward: entry=27'10 (0'0) error 1:e7f24557:::smithi13140796-34:head by client.4540.0:218 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate - purged_snaps [2~1] removed_snaps [2~1,4~1] 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:34:06.663+0000 7fe2c425f700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] recalc_readable_until peer osd.5 ruub 0s 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] recalc_readable_until peer osd.7 ruub 0s 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] recalc_readable_until readable_until[_ub] 0s (sent 46.119144108s) 2019-12-07T19:34:06.663+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.7s0 1.7s0 (query:29 sent:29 1.7s3( v 27'10 (0'0,27'10] local-lis/les=26/27 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) 3->0 ([18,27] all_participants=0(3),1(0),2(5),3(2),4(1),4(2),6(1),7(4) intervals=([18,23] acting 0(3),1(0),2(5),4(2),6(1),7(4)))) e29/29) v1 -- 0x558e846f9500 con 0x558e818d7c00 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.1(2) 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954225517s) 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.1(2) is up to date, queueing in pending_activators 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e81a0a600 seq 31 2019-12-07T19:34:06.663+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.1(2) 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.954225517s) uptodate 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.3(5) 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956606539s) 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.3(5) is up to date, queueing in pending_activators 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.3(5) 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956606539s) uptodate 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a600 cost 5526 (1 ios * 4000 + 1526 bytes) 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a600 onodes shared_blobs 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.4(1) 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956942681s) 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a600 allocated 0x[] released 0x[] 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.4(1) is up to date, queueing in pending_activators 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 prepare 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a600 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 io_done 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.4(1) 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.956942681s) uptodate 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558e8122e000 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.5(3) 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.5(3) is up to date, queueing in pending_activators 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.5(3) 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) uptodate 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558e81a9b000 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.663+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.7(4) 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) 2019-12-07T19:34:06.663+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.7(4) is up to date, queueing in pending_activators 2019-12-07T19:34:06.663+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.104938842s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.663+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.663+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.104938842s int 16s)) prio 255 cost 10 e29) pg 0x558e83c78000 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate peer osd.7(4) 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=0/18 les/c/f=0/19/0 sis=28) uptodate 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate setting up missing_loc from shard 0(0) 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate setting up missing_loc from shard 1(2) 2019-12-07T19:34:06.663+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate setting up missing_loc from shard 3(5) 2019-12-07T19:34:06.663+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs1 28 -> 29 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate setting up missing_loc from shard 4(1) 2019-12-07T19:34:06.663+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate setting up missing_loc from shard 5(3) 2019-12-07T19:34:06.663+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] activate setting up missing_loc from shard 7(4) 2019-12-07T19:34:06.663+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.663+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs1_head options 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 unknown mbc={} ps=[2~1]] needs_recovery is recovered 2019-12-07T19:34:06.663+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.663+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.663+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] release_backoffs [1:40000000::::0,1:50000000::::head) 2019-12-07T19:34:06.663+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.663+0000 7fe2ca26b700 20 osd.0 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.663+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(3),7(4) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:06.663+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 0(0) primary objects 4 missing 0 2019-12-07T19:34:06.663+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(2) objects 4 missing 0 2019-12-07T19:34:06.663+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.663+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.104938842s int 16s) 2019-12-07T19:34:06.663+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 4 missing 0 2019-12-07T19:34:06.664+0000 7fe2cc26f700 20 osd.0 29 agent_entry empty queue 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(1) objects 4 missing 0 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] state: got info from osd.4(0) 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.104938842s int 16s) 2019-12-07T19:34:06.664+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.131453992s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] peer_clock_delta_lb 18446744073.703788851s -> ruub 46.110701607s 2019-12-07T19:34:06.664+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.131453992s int 16s)) prio 255 cost 10 e29) pg 0x558e83d07000 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.110701607s 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(3) objects 4 missing 0 2019-12-07T19:34:06.664+0000 7fe2ca26b700 5 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.664722+0000 5 0.000077 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats no peer_missing found for 5(4) 2019-12-07T19:34:06.664+0000 7fe2ca26b700 5 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 1 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946296912s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.664+0000 7fe2c5261700 20 osd.0:5.update_pg_epoch 1.5s4 28 -> 29 2019-12-07T19:34:06.664+0000 7fe2ca26b700 5 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(4) objects 4 missing 0 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 28 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946296912s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] handle_advance_map [2,1,3,4,0,5]/[2,1,3,4,0,5] -- 2/2 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(3),7(4)=4} 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] activate - no missing, moving last_complete 21'12 -> 21'12 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946296912s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946296912s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.5s4_head options 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946296912s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 29 send_incremental_map 28 -> 29 to 0x558e818fe400 v2:172.21.15.109:6810/36938 2019-12-07T19:34:06.664+0000 7fe2c5261700 20 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946296912s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:06.664+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- osd_map(29..29 src has 1..29) v4 -- 0x558e8419cb40 con 0x558e818fe400 2019-12-07T19:34:06.664+0000 7fe2c5261700 20 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946296912s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(4) missing= 0 2019-12-07T19:34:06.664+0000 7fe2c5261700 20 osd.0 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 29 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 28 -> 29 (shared) 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946296912s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.946296912s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.664+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:29 sent:29 1.bs1( v 21'12 (0'0,21'12] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 1->0 ([18,27] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e29/29) v1 -- 0x558e84704e00 con 0x558e818fe400 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:06.664+0000 7fe2caa6c700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 28:70 2019-12-07T19:34:06.664+0000 7fe2c5261700 20 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e811fb800 seq 10 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.131453992s int 16s) 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._biginfo' <- _biginfo 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._epoch' <- _epoch 2019-12-07T19:34:06.664+0000 7fe2caa6c700 5 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] enter Started/Primary/Active/Activating 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] state: got info from osd.2(0) 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fb800 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fb800 onodes shared_blobs 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fb800 allocated 0x[] released 0x[] 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.131453992s int 16s) 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 prepare 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fb800 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 io_done 2019-12-07T19:34:06.664+0000 7fe2c5261700 20 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] peer_clock_delta_lb 0.016836991s -> ruub 46.114617001s 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2c5261700 20 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.114617001s 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.105685108s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.105685108s int 16s)) prio 255 cost 10 e29) pg 0x558e81231000 2019-12-07T19:34:06.664+0000 7fe2c5261700 5 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.664862+0000 5 0.000122 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] requeue_map_waiters client.4540 0x558e84176b00 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8) prio 63 cost 0 e29) 2019-12-07T19:34:06.664+0000 7fe2c5261700 5 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.664+0000 7fe2c5261700 5 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 4(1) notify: (query:29 sent:29 1.2s1( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.955112918s) 1->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.3s5 28 -> 29 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown NOTIFY pruub 44.064407826s@ mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] state: Active: got notify from 4(1), already have info from that osd, ignoring 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.064407826s@ mbc={} ps=[2~1]] activate - no missing, moving last_complete 27'8 -> 27'8 2019-12-07T19:34:06.664+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 -- 0x558e83c46700 con 0x558e818d7c00 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.064407826s@ mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.664+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.3s5_head options 2019-12-07T19:34:06.664+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 -- 0x558e83ebca00 con 0x558e818d7000 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.664+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 -- 0x558e83ebc300 con 0x558e818fe400 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 29 send_incremental_map 28 -> 29 to 0x558e818d6800 v2:172.21.15.109:6802/36936 2019-12-07T19:34:06.664+0000 7fe2c5261700 10 osd.0 29 send_incremental_map 28 -> 29 to 0x558e818d6000 v2:172.21.15.131:6804/37077 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.664+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd_map(29..29 src has 1..29) v4 -- 0x558e8419d200 con 0x558e818d6800 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 29 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 28 -> 29 (shared) 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.105685108s int 16s) 2019-12-07T19:34:06.664+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 -- 0x558e83ebbc00 con 0x558e818d6800 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] state: got info from osd.4(0) 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 29 send_incremental_map 28 -> 29 to 0x558e818fe000 v2:172.21.15.109:6826/36939 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.105685108s int 16s) 2019-12-07T19:34:06.664+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- osd_map(29..29 src has 1..29) v4 -- 0x558e8419d8c0 con 0x558e818fe000 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 29 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 28 -> 29 (shared) 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] peer_clock_delta_lb 18446744073.703788851s -> ruub 46.111447873s 2019-12-07T19:34:06.664+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_info2(1.2s4 1.2s4 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) e29/29) v1 -- 0x558e80ff3100 con 0x558e818fe000 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.111447873s 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:06.664+0000 7fe2ca26b700 5 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.664983+0000 5 0.000090 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e84558000 seq 31 2019-12-07T19:34:06.664+0000 7fe2ca26b700 5 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.664+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:06.664+0000 7fe2ca26b700 5 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:06.664+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xe0796 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown NOTIFY mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.664+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.664+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:06.664+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.664+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84558000 cost 5607 (1 ios * 4000 + 1607 bytes) 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84558000 onodes shared_blobs 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84558000 allocated 0x[] released 0x[] 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] activate - no missing, moving last_complete 23'10 -> 23'10 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558000 prepare 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84558000 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558000 io_done 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 7(4) notify: (query:29 sent:29 1.2s4( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.959400255s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.664+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.3s0 1.3s0 (query:29 sent:29 1.3s5( v 23'10 (0'0,23'10] local-lis/les=0/0 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 5->0 ([18,27] all_participants=0(5),1(1),2(2),4(0),5(3),6(5),7(4) intervals=([26,27] acting 1(1),2(2),4(0),5(3),7(4)))) e29/29) v1 -- 0x558e84705500 con 0x558e818fe400 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] state: Active: got notify from 7(4), already have info from that osd, ignoring 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e84559800 seq 10 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._biginfo' <- _biginfo 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._epoch' <- _epoch 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(3) notify: (query:29 sent:29 1.2s3( v 27'12 (0'0,27'12] local-lis/les=0/0 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957841255s) 3->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84559800 cost 5425 (1 ios * 4000 + 1425 bytes) 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84559800 onodes shared_blobs 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84559800 allocated 0x[] released 0x[] 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] state: Active: got notify from 5(3), already have info from that osd, ignoring 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559800 prepare 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84559800 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559800 io_done 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558e81231000 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 5(4) notify: (query:29 sent:29 1.2s4( v 23'10 (0'0,23'10] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.957764085s) 4->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] state: Active: got notify from 5(4), already have info from that osd, ignoring 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558e83c78000 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 3(5) notify: (query:29 sent:29 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953485642s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 3(5) notify: (query:29 sent:29 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953485642s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558e81928000 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 3(5) notify: (query:29 sent:29 1.2s5( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.953485642s) 5->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] state: Active: got notify from 3(5), already have info from that osd, ignoring 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 1(2) notify: (query:29 sent:29 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950531088s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process 0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29)> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 1(2) notify: (query:29 sent:29 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950531088s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs3 28 -> 29 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MNotifyRec 1.2s0 from 1(2) notify: (query:29 sent:29 1.2s2( v 27'12 (0'0,27'12] local-lis/les=26/27 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.950531088s) 2->0 ([18,27] all_participants=0(0),1(2),3(5),4(1),5(3),5(4),6(3),7(4) intervals=([18,23] acting 0(0),1(2),3(5),4(1),5(4),6(3)))) features: 0x3f01cfb8ffadffff +create_info 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 28 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] state: Active: got notify from 1(2), already have info from that osd, ignoring 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115034256s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115034256s int 16s)) prio 255 cost 10 e29) pg 0x558e8193b000 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.664+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs3_head options 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951614038s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] handle_advance_map: 29 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.as3 28 -> 29 2019-12-07T19:34:06.664+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (34) on file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]), flushing log 2019-12-07T19:34:06.664+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.664+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 28 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951614038s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] handle_advance_map [1,4,2,0,5,3]/[1,4,2,0,5,3] -- 1/1 2019-12-07T19:34:06.664+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 34 len 0x49 crc 0x1b5a5c9a) 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.664+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x21000~1000 to file(ino 1 size 0x21000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951614038s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] state: Started advmap 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.955800138s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.664+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x21000 0x21000~1000 to file(ino 1 size 0x21000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:06.664+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x22000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:06.664+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x21000 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951614038s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.664+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951614038s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] plpg_on_pool_change 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.664+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951614038s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951614038s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951614038s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] handle_activate_map: 29 2019-12-07T19:34:06.664+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_notify2(1.bs0 1.bs0 (query:29 sent:29 1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) 3->0 ([18,27] all_participants=0(1),0(3),1(4),2(2),4(0),5(5),6(1),7(3) intervals=([18,23] acting 0(3),1(4),2(2),4(0),5(5),6(1)))) e29/29) v1 -- 0x558e84705c00 con 0x558e818fe400 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=14.951614038s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] activate_map 2019-12-07T19:34:06.664+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.664+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x22000 2019-12-07T19:34:06.664+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:06.664+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) pg_lease(ru 0s ub 46.115034256s int 16s) 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] state: got info from osd.1(0) 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] proc_lease pg_lease(ru 0s ub 46.115034256s int 16s) 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] peer_clock_delta_lb 0.001072657s -> ruub 46.113961599s 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] proc_lease readable_until_ub now 46.113961599s 2019-12-07T19:34:06.664+0000 7fe2caa6c700 5 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.665282+0000 5 0.000078 2019-12-07T19:34:06.664+0000 7fe2caa6c700 5 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] enter Started/ReplicaActive 2019-12-07T19:34:06.664+0000 7fe2caa6c700 5 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] enter Started/ReplicaActive/RepNotRecovering 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown NOTIFY pruub 44.064869888s@ mbc={} ps=[2~1]] state: In ReplicaActive, about to call activate 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] activate - no missing, moving last_complete 21'2 -> 21'2 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] state: Activate Finished 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_notify2(1.as0 1.as0 (query:29 sent:29 1.as3( v 21'2 (0'0,21'2] local-lis/les=26/27 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) 3->0 ([18,27] all_participants=0(3),1(0),2(2),3(5),4(1),4(5),5(4),6(1) intervals=([18,23] acting 0(3),1(0),2(2),4(5),5(4),6(1)))) e29/29) v1 -- 0x558e83ebb500 con 0x558e818d7c00 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e84558300 seq 17 2019-12-07T19:34:06.664+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84558300 cost 5447 (1 ios * 4000 + 1447 bytes) 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84558300 onodes shared_blobs 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84558300 allocated 0x[] released 0x[] 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558300 prepare 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84558300 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558300 io_done 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558e8193b000 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8) prio 63 cost 0 e29) queued 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8) prio 63 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e84176b00 prio 63 cost 0 latency 0.005317 osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8 pg pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.664+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e84176b00 finish 2019-12-07T19:34:06.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.664+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.664+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.664+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.664+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.664+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.664+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558e81938000 2019-12-07T19:34:06.664+0000 7fe2c5a62700 10 osd.0 pg_epoch: 28 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_advance_map: 29 2019-12-07T19:34:06.664+0000 7fe2c5a62700 20 osd.0:4.update_pg_epoch 1.4s1 28 -> 29 2019-12-07T19:34:06.664+0000 7fe2c5a62700 10 osd.0 pg_epoch: 28 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:06.664+0000 7fe2c5a62700 10 osd.0 pg_epoch: 29 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:06.664+0000 7fe2c5a62700 10 osd.0 pg_epoch: 29 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.664+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:34:06.664+0000 7fe2c5a62700 10 osd.0 pg_epoch: 29 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:06.664+0000 7fe2c5a62700 20 osd.0 pg_epoch: 29 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:06.664+0000 7fe2c5a62700 20 osd.0 pg_epoch: 29 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:06.664+0000 7fe2c5a62700 20 osd.0 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.664+0000 7fe2c5a62700 10 osd.0 pg_epoch: 29 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_activate_map: 29 2019-12-07T19:34:06.665+0000 7fe2c5a62700 10 osd.0 pg_epoch: 29 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:06.665+0000 7fe2c5a62700 20 osd.0 pg_epoch: 29 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.665+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:06.665+0000 7fe2c5a62700 10 osd.0 pg_epoch: 29 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.665+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:06.665+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:06.665+0000 7fe2c5261700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- osd_map(29..29 src has 1..29) v4 -- 0x558e8178f8c0 con 0x558e818d6000 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 osd.0 29 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 28 -> 29 (shared) 2019-12-07T19:34:06.665+0000 7fe2c5261700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_notify2(1.5s0 1.5s0 (query:29 sent:29 1.5s4( v 27'8 (0'0,27'8] local-lis/les=0/0 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) 4->0 ([18,27] all_participants=0(4),1(1),1(4),2(0),3(2),4(3),5(5),6(1) intervals=([18,23] acting 1(4),2(0),3(2),4(3),5(5),6(1)))) e29/29) v1 -- 0x558e83ebd800 con 0x558e818d6000 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e8417b500 seq 12 2019-12-07T19:34:06.665+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._biginfo' <- _biginfo 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._epoch' <- _epoch 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417b500 cost 5617 (1 ios * 4000 + 1617 bytes) 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417b500 onodes shared_blobs 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417b500 allocated 0x[] released 0x[] 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 prepare 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417b500 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 io_done 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558e83d07000 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.064407826s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 NullEvt) prio 255 cost 10 e29) pg 0x558e819e7000 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 osd.0 pg_epoch: 28 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_advance_map: 29 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 osd.0:5.update_pg_epoch 1.ds5 28 -> 29 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 osd.0 pg_epoch: 28 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:06.665+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 osd.0 pg_epoch: 29 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 osd.0 pg_epoch: 29 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 osd.0 29 advance_pg new pool opts old pool opts 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_activate_map: 29 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:06.665+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 osd.0 pg_epoch: 29 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 29 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 NullEvt 2019-12-07T19:34:06.665+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:06.665+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:06.665+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 54 ==== pg_info2(1.2s0 1.2s0 1.2s3( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.957841255s) e29/29) v1 ==== 1173+0+0 (unknown 0 0 0) 0x558e80ff3100 con 0x558e818d6800 2019-12-07T19:34:06.665+0000 7fe2e753b700 15 osd.0 29 enqueue_peering_evt 1.2s0 epoch_sent: 29 epoch_requested: 29 MInfoRec from 5(3) info: 1.2s3( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.957841255s) pg_lease_ack(ruub 46.119144108s) 2019-12-07T19:34:06.665+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 5(3) info: 1.2s3( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.957841255s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.665+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 34 ==== pg_info2(1.2s0 1.2s0 1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.959400255s) e29/29) v1 ==== 1173+0+0 (unknown 0 0 0) 0x558e80ff3100 con 0x558e818fe000 2019-12-07T19:34:06.665+0000 7fe2e753b700 15 osd.0 29 enqueue_peering_evt 1.2s0 epoch_sent: 29 epoch_requested: 29 MInfoRec from 7(4) info: 1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.959400255s) pg_lease_ack(ruub 46.119144108s) 2019-12-07T19:34:06.665+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 7(4) info: 1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.959400255s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 5(3) info: 1.2s3( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.957841255s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 5(3) info: 1.2s3( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.957841255s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.665+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 5(3) info: 1.2s3( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.957841255s) pg_lease_ack(ruub 46.119144108s) 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.5 ruub 46.119144108s 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.7 ruub 0s 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until readable_until[_ub] 0s (sent 46.119144108s) 2019-12-07T19:34:06.665+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] state: peer osd.5(3) activated and committed 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(3),7(4) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 0(0) primary objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(2) objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(1) objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(3) objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats no peer_missing found for 5(4) 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 1 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(4) objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(3),7(4)=4} 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(4) missing= 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.665+0000 7fe2caa6c700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:71 2019-12-07T19:34:06.665+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 7(4) info: 1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.959400255s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 7(4) info: 1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.959400255s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.665+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 7(4) info: 1.2s4( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.959400255s) pg_lease_ack(ruub 46.119144108s) 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.1 ruub 0s 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.5 ruub 46.119144108s 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.7 ruub 46.119144108s 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until readable_until[_ub] 0s (sent 46.119144108s) 2019-12-07T19:34:06.665+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] state: peer osd.7(4) activated and committed 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(3),7(4) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 0(0) primary objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(2) objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(1) objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(3) objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats no peer_missing found for 5(4) 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 1 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(4) objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(3),7(4)=4} 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(4) missing= 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.665+0000 7fe2caa6c700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:72 2019-12-07T19:34:06.665+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.665+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.665+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 196 ==== pg_info2(1.2s0 1.2s0 1.2s2( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.950531088s) e29/29) v1 ==== 1173+0+0 (unknown 0 0 0) 0x558e83ebca00 con 0x558e818d7c00 2019-12-07T19:34:06.665+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.2s0 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(2) info: 1.2s2( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.950531088s) pg_lease_ack(ruub 46.119144108s) 2019-12-07T19:34:06.665+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(2) info: 1.2s2( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.950531088s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(2) info: 1.2s2( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.950531088s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(2) info: 1.2s2( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.950531088s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.665+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(2) info: 1.2s2( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.950531088s) pg_lease_ack(ruub 46.119144108s) 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.1 ruub 46.119144108s 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.5 ruub 46.119144108s 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.7 ruub 46.119144108s 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.3 ruub 0s 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until readable_until[_ub] 0s (sent 46.119144108s) 2019-12-07T19:34:06.665+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] state: peer osd.1(2) activated and committed 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(3),7(4) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 0(0) primary objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(2) objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 4 missing 0 2019-12-07T19:34:06.665+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(1) objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.666+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 110 ==== pg_info2(1.2s0 1.2s0 1.2s5( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.953485642s) e29/29) v1 ==== 1173+0+0 (unknown 0 0 0) 0x558e83ebca00 con 0x558e818d7000 2019-12-07T19:34:06.666+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.2s0 epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(5) info: 1.2s5( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.953485642s) pg_lease_ack(ruub 46.119144108s) 2019-12-07T19:34:06.666+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(5) info: 1.2s5( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.953485642s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(3) objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats no peer_missing found for 5(4) 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 1 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(4) objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(5) info: 1.2s5( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.953485642s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(3),7(4)=4} 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(4) missing= 0 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.666+0000 7fe2c6a64700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:73 2019-12-07T19:34:06.666+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(5) info: 1.2s5( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.953485642s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.666+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(5) info: 1.2s5( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.953485642s) pg_lease_ack(ruub 46.119144108s) 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.4 ruub 0s 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.1 ruub 46.119144108s 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.5 ruub 46.119144108s 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.7 ruub 46.119144108s 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.3 ruub 46.119144108s 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until readable_until[_ub] 0s (sent 46.119144108s) 2019-12-07T19:34:06.666+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] state: peer osd.3(5) activated and committed 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(3),7(4) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 0(0) primary objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(2) objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(1) objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(3) objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats no peer_missing found for 5(4) 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 1 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(4) objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(3),7(4)=4} 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(4) missing= 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.666+0000 7fe2caa6c700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:74 2019-12-07T19:34:06.666+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.666+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.666+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 110 ==== pg_info2(1.2s0 1.2s0 1.2s1( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.955112918s) e29/29) v1 ==== 1173+0+0 (unknown 0 0 0) 0x558e84705c00 con 0x558e818fe400 2019-12-07T19:34:06.666+0000 7fe2e6d3a700 15 osd.0 29 enqueue_peering_evt 1.2s0 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(1) info: 1.2s1( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.955112918s) pg_lease_ack(ruub 46.119144108s) 2019-12-07T19:34:06.666+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(1) info: 1.2s1( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.955112918s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(1) info: 1.2s1( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.955112918s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(1) info: 1.2s1( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.955112918s) pg_lease_ack(ruub 46.119144108s)) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.666+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(1) info: 1.2s1( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.955112918s) pg_lease_ack(ruub 46.119144108s) 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.002100 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.4 ruub 46.119144108s 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.1 ruub 46.119144108s 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.5 ruub 46.119144108s 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.7 ruub 46.119144108s 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until peer osd.3 ruub 46.119144108s 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] recalc_readable_until readable_until[_ub] 46.119144108s (sent 46.119144108s) 2019-12-07T19:34:06.666+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] state: peer osd.4(1) activated and committed 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 34 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(3),7(4) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 0(0) primary objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(2) objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4d500,0x558e83c56c00,0x558e8417a600,0x558e83d4c900> 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 kv_submitted 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4d500 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d500 finishing 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4d500 onodes 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4d500 done 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(1) objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4d500 [] 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 kv_submitted 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56c00 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(3) objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56c00 finishing 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56c00 onodes 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56c00 done 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56c00 [] 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats no peer_missing found for 5(4) 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 kv_submitted 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417a600 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 finishing 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417a600 onodes 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 1 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a600 done 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417a600 [] 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 kv_submitted 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c900 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c900 finishing 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c900 onodes 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(4) objects 4 missing 0 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c900 done 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c900 [] 2019-12-07T19:34:06.666+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.666+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(3),7(4)=4} 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 4 cleaned 0 in 0.003195120s (0.000000104s flush + 0.003195016s kv commit) 2019-12-07T19:34:06.666+0000 7fe2cb26d700 15 osd.0 29 enqueue_peering_evt 1.1s1 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:06.666+0000 7fe2cb26d700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 7 submitting 7 deferred done 0 stable 0 2019-12-07T19:34:06.666+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1387 < min_flush_size 524288 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2769 < min_flush_size 524288 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(4) missing= 0 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4060 < min_flush_size 524288 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5511 < min_flush_size 524288 2019-12-07T19:34:06.666+0000 7fe2c6a64700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:75 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:06.666+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 6780 < min_flush_size 524288 2019-12-07T19:34:06.666+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8071 < min_flush_size 524288 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 9532 < min_flush_size 524288 2019-12-07T19:34:06.666+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 9555 < min_flush_size 524288 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 9555 < min_flush_size 524288 2019-12-07T19:34:06.666+0000 7fe2cba6e700 15 osd.0 29 enqueue_peering_evt 1.8s5 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.666+0000 7fe2cba6e700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xe0796~2553 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.666+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.666+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xe0796 0xe0796~2553 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.666+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.666+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558e841a4000 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb0796 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x796 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xce9 and padding block with 0x317 2019-12-07T19:34:06.666+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown pruub 44.065109905s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.666+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.666+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 crt=27'23 unknown pruub 44.065109905s@ mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.666+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:06.666+0000 7fe2c8a68700 15 osd.0 29 enqueue_peering_evt 1.es4 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.666+0000 7fe2c8a68700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.666+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.666+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.666+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.8s0 1.8s0 1.8s5( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.946615610s) e29/29) v1 -- 0x558e846fb100 con 0x558e818d7c00 2019-12-07T19:34:06.666+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.666+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558e81a98000 2019-12-07T19:34:06.666+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.666+0000 7fe2c8a68700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.666+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.666+0000 7fe2c8a68700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 crt=21'4 unknown pruub 44.060997103s@ mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.666+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xe2ce9 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.666+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.666+0000 7fe2c8a68700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.es0 1.es0 1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.942492559s) e29/29) v1 -- 0x558e83c9ce00 con 0x558e818d7c00 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 7 cleaned 0 in 0.000183942s (0.000000090s flush + 0.000183852s kv commit) 2019-12-07T19:34:06.666+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.666+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.666+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.667+0000 7fe2cb26d700 15 osd.0 29 enqueue_peering_evt 1.9s3 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2cb26d700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.667+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.667+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558e819e4000 2019-12-07T19:34:06.667+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'7 unknown pruub 44.059111212s@ mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.667+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:06.667+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.1s0 1.1s0 1.1s1( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.940748925s) e29/29) v1 -- 0x558e81228000 con 0x558e818d7000 2019-12-07T19:34:06.667+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.667+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558e8192b000 2019-12-07T19:34:06.667+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.060646370s@ mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.667+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:06.667+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.9s0 1.9s0 1.9s3( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.941957470s) e29/29) v1 -- 0x558e8404bc00 con 0x558e818d7c00 2019-12-07T19:34:06.667+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.667+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4cc00,0x558e81a0a600,0x558e811fb800,0x558e84558000,0x558e84559800,0x558e84558300,0x558e8417b500> 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 kv_submitted 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4cc00 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 finishing 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4cc00 onodes 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4cc00 done 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4cc00 [] 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 kv_submitted 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a600 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a600 finishing 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a600 onodes 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a600 done 2019-12-07T19:34:06.667+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a600 [] 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 kv_submitted 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fb800 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb800 finishing 2019-12-07T19:34:06.667+0000 7fe2c8267700 15 osd.0 29 enqueue_peering_evt 1.fs1 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fb800 onodes 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb800 done 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:06.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fb800 [] 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.667+0000 7fe2c8267700 15 osd.0 29 enqueue_peering_evt 1.7s3 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558000 kv_submitted 2019-12-07T19:34:06.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84558000 2019-12-07T19:34:06.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558000 finishing 2019-12-07T19:34:06.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84558000 onodes 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84558000 done 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:06.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84558000 [] 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558e81a9b000 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559800 kv_submitted 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84559800 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559800 finishing 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84559800 onodes 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84559800 done 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84559800 [] 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558300 kv_submitted 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84558300 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558300 finishing 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84558300 onodes 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84558300 done 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84558300 [] 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 kv_submitted 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417b500 2019-12-07T19:34:06.667+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 finishing 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417b500 onodes 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b500 done 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417b500 [] 2019-12-07T19:34:06.667+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.667+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.667+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.059243341s@ mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.667+0000 7fe2caa6c700 15 osd.0 29 enqueue_peering_evt 1.2s0 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.667+0000 7fe2caa6c700 15 osd.0 29 enqueue_peering_evt 1.as3 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.667+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.fs0 1.fs0 1.fs1( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.940456110s) e29/29) v1 -- 0x558e81a58e00 con 0x558e818d7000 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.667+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558e8122e000 2019-12-07T19:34:06.667+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] state: _activate_committed 29 peer_activated now 0(0),1(2),3(5),4(1),5(3),7(4) last_interval_started 26 last_epoch_started 27 same_interval_since 28 2019-12-07T19:34:06.667+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] state: all_activated_and_committed 2019-12-07T19:34:06.667+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=27'10 unknown pruub 44.061193946s@ mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.667+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(3),7(4) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:06.667+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.7s0 1.7s0 1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.942150657s) e29/29) v1 -- 0x558e81a59500 con 0x558e818d7c00 2019-12-07T19:34:06.667+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 0(0) primary objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(2) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2c9269700 15 osd.0 29 enqueue_peering_evt 1.5s4 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2c9269700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.667+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558e83d07000 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(1) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(3) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2c9269700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.064407826s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats no peer_missing found for 5(4) 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 1 2019-12-07T19:34:06.667+0000 7fe2c9269700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 crt=27'8 unknown pruub 44.064407826s@ mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(4) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(3),7(4)=4} 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:06.667+0000 7fe2c9269700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_info2(1.5s0 1.5s0 1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.944908336s) e29/29) v1 -- 0x558e84704700 con 0x558e818d6000 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:06.667+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(4) missing= 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 activating mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] state: mnow 30.123101982s >= prior_readable_until_ub 0s 2019-12-07T19:34:06.667+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] share_pg_info 2019-12-07T19:34:06.667+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 -- 0x558e83c9c700 con 0x558e818d7c00 2019-12-07T19:34:06.667+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.667+0000 7fe2ca26b700 15 osd.0 29 enqueue_peering_evt 1.bs1 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2ca26b700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.667+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 -- 0x558e83c9dc00 con 0x558e818d7000 2019-12-07T19:34:06.667+0000 7fe2ca26b700 15 osd.0 29 enqueue_peering_evt 1.3s5 epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2ca26b700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) 2019-12-07T19:34:06.667+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.667+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558e83c78000 2019-12-07T19:34:06.667+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 -- 0x558e83c9ea00 con 0x558e818fe400 2019-12-07T19:34:06.667+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 crt=21'12 unknown mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.667+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 -- 0x558e83c48300 con 0x558e818d6800 2019-12-07T19:34:06.667+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.667+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.bs0 1.bs0 1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 -- 0x558e84706300 con 0x558e818fe400 2019-12-07T19:34:06.667+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_info2(1.2s4 1.2s4 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 -- 0x558e83c46e00 con 0x558e818fe000 2019-12-07T19:34:06.667+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.667+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.667+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558e81231000 2019-12-07T19:34:06.667+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(3),7(4) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:06.667+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 crt=23'10 unknown mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 0(0) primary objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(2) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.3s0 1.3s0 1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 -- 0x558e84706a00 con 0x558e818fe400 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.667+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(1) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(3) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats no peer_missing found for 5(4) 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 1 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(4) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(3),7(4)=4} 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(4) missing= 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.667+0000 7fe2caa6c700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:76 2019-12-07T19:34:06.667+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] check_local 2019-12-07T19:34:06.667+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] checking 1:4c9e444c:::smithi13140796-48:head at 23'10 2019-12-07T19:34:06.667+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:4c9e444c:::smithi13140796-48:head# 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] requeue_op 0x558e84176b00 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8) prio 63 cost 0 e29) 2019-12-07T19:34:06.667+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] needs_recovery is recovered 2019-12-07T19:34:06.667+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] needs_backfill does not need backfill 2019-12-07T19:34:06.667+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] activate all replicas clean, no recovery 2019-12-07T19:34:06.667+0000 7fe2caa6c700 15 osd.0 29 enqueue_peering_evt 1.2s0 epoch_sent: 29 epoch_requested: 29 AllReplicasRecovered 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 AllReplicasRecovered) prio 255 cost 10 e29) 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(3),7(4) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 0(0) primary objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(2) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(1) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(3) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats no peer_missing found for 5(4) 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 1 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(4) objects 4 missing 0 2019-12-07T19:34:06.667+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(3),7(4)=4} 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(4) missing= 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.668+0000 7fe2caa6c700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:77 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] hit_set_clear 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] agent_stop 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e84558600 seq 32 2019-12-07T19:34:06.668+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:06.668+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84558600 cost 5332 (1 ios * 4000 + 1332 bytes) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84558600 onodes shared_blobs 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84558600 allocated 0x[] released 0x[] 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558600 prepare 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84558600 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558600 io_done 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29) prio 255 cost 10 e29) pg 0x558e8193b000 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 ActivateCommitted from 29 processed at 29 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 crt=21'2 unknown pruub 44.064869888s@ mbc={} ps=[2~1]] state: react 29 telling primary 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.668+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.as0 1.as0 1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 pruub=13.944939721s) e29/29) v1 -- 0x558e83c49100 con 0x558e818d7c00 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 AllReplicasRecovered) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 AllReplicasRecovered) prio 255 cost 10 e29) pg 0x558e81075000 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 AllReplicasRecovered 2019-12-07T19:34:06.668+0000 7fe2caa6c700 5 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] exit Started/Primary/Active/Activating 2019-12-07T19:34:06.668878+0000 14 0.001994 2019-12-07T19:34:06.668+0000 7fe2caa6c700 5 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] enter Started/Primary/Active/Recovered 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] needs_recovery is recovered 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats actingset 0(0),1(2),3(5),4(1),5(3),7(4) upset 0(0),1(2),3(5),4(1),5(3),7(4) acting_recovery_backfill 0(0),1(2),3(5),4(1),5(3),7(4) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 0(0) primary objects 4 missing 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 1(2) objects 4 missing 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 3(5) objects 4 missing 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 4(1) objects 4 missing 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(3) objects 4 missing 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats no peer_missing found for 5(4) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 5(4) objects 3 missing 1 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats shard 7(4) objects 4 missing 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats object_location_counts {0(0),1(2),3(5),4(1),5(3),7(4)=4} 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 0(0) missing= 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 1(2) missing= 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 3(5) missing= 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 4(1) missing= 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 5(3) missing= 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats missing shard 7(4) missing= 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats degraded 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] update_calc_stats misplaced 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.668+0000 7fe2caa6c700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:78 2019-12-07T19:34:06.668+0000 7fe2caa6c700 5 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] exit Started/Primary/Active/Recovered 2019-12-07T19:34:06.669031+0000 0 0.000000 2019-12-07T19:34:06.668+0000 7fe2caa6c700 5 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 pi=[18,28)/1 crt=27'12 lcod 23'10 mlcod 0'0 active mbc={} trimq=[4~1] ps=[2~1]] enter Started/Primary/Active/Clean 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] share_pg_info 2019-12-07T19:34:06.668+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558e83c48a00 con 0x558e818d7c00 2019-12-07T19:34:06.668+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558e8122b100 con 0x558e818d7000 2019-12-07T19:34:06.668+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558e81229500 con 0x558e818fe400 2019-12-07T19:34:06.668+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558e81229c00 con 0x558e818d6800 2019-12-07T19:34:06.668+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_info2(1.2s4 1.2s4 1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558e8404ca00 con 0x558e818fe000 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.668+0000 7fe2caa6c700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:79 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] exit NotTrimming 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] enter Trimming 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] enter Trimming/WaitReservation 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.668+0000 7fe2caa6c700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:80 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] finish_recovery 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] clear_recovery_state 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e84558900 seq 33 2019-12-07T19:34:06.668+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84558900 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84558900 onodes shared_blobs 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84558900 allocated 0x[] released 0x[] 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558900 prepare 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84558900 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558900 io_done 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8) prio 63 cost 0 e29) queued 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8) prio 63 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e84176b00 prio 63 cost 0 latency 0.009124 osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8 pg pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] _handle_message: osd_op(client.4540.0:249 1.2s0 1.5e84fb82 (undecoded) ondisk+write+peerstat_old+known_if_redirected e29) v8 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_op: op osd_op(client.4540.0:249 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 6=[6] ondisk+write+peerstat_old+known_if_redirected e29) v8 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_op osd_op(client.4540.0:249 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 6=[6] ondisk+write+peerstat_old+known_if_redirected e29) v8 may_write -> write-ordered flags ondisk+write+peerstat_old+known_if_redirected 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] get_object_context: obc NOT found in cache: 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:06.668+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# _ 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# _ = 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# snapset 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# snapset = 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] populate_obc_watchers 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:06.668+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattrs 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattrs 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# = 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] get_object_context: creating obc from disk: 0x558e81222900 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] get_object_context: 0x558e81222900 1:41df217a:::smithi13140796-39:head rwstate(none n=0 w=0) oi: 1:41df217a:::smithi13140796-39:head(27'12 client.4540.0:231 whiteout|dirty s 0 uv 12 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e83dcc6c0 snapset: 4=[]:{4=[4]} 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] find_object_context 1:41df217a:::smithi13140796-39:head @head oi=1:41df217a:::smithi13140796-39:head(27'12 client.4540.0:231 whiteout|dirty s 0 uv 12 alloc_hint [0 0 0]) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 25 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_op oi 1:41df217a:::smithi13140796-39:head(27'12 client.4540.0:231 whiteout|dirty s 0 uv 12 alloc_hint [0 0 0]) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_op obc obc(1:41df217a:::smithi13140796-39:head rwstate(write n=1 w=0)) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] execute_ctx 0x558e8404d100 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] execute_ctx 1:41df217a:::smithi13140796-39:head [delete] ov 27'12 av 29'13 snapc 6=[6] snapset 4=[]:{4=[4]} 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_osd_op 1:41df217a:::smithi13140796-39:head [delete] 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_osd_op delete 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] _delete_oid has or will have clones; will whiteout 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] _delete_oid 1:41df217a:::smithi13140796-39:head whiteout=1 no_whiteout=0 try_no_whiteout=0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] do_osd_ops error: (2) No such file or directory 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] execute_ctx alloc reply 0x558e8417c000 result -2 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] calc_trim_to_aggressive limit = 27'12 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] op order client.4540 tid 249 (first) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] execute_ctx update_log_only -- result=-2 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] record_write_error r=-2 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] submit_log_entries 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] new_repop: repgather(0x558e83dcc5a0 29'13 rep_tid=25 committed?=0 r=-2) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 27'12 (0'0,27'12] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] merge_new_log_entries 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 update missing, append 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 lua=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] append_log_entries_update_missing trim_to bool = 1 trim_to = 0'0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'13, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 lua=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] merge_new_log_entries peer_missing for 1(2) = missing(0 may_include_deletes = 1) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 lua=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] merge_new_log_entries peer_missing for 3(5) = missing(0 may_include_deletes = 1) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 lua=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] merge_new_log_entries peer_missing for 4(1) = missing(0 may_include_deletes = 1) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 lua=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] merge_new_log_entries peer_missing for 5(3) = missing(0 may_include_deletes = 1) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 lua=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] merge_new_log_entries peer_missing for 7(4) = missing(0 may_include_deletes = 1) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_update_log_missing(1.2s2 epoch 29/28 rep_tid 25 entries 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 -- 0x558e83c678c0 con 0x558e818d7c00 2019-12-07T19:34:06.668+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_update_log_missing(1.2s5 epoch 29/28 rep_tid 25 entries 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 -- 0x558e83c66240 con 0x558e818d7000 2019-12-07T19:34:06.668+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_update_log_missing(1.2s1 epoch 29/28 rep_tid 25 entries 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 -- 0x558e84409d40 con 0x558e818fe400 2019-12-07T19:34:06.668+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_update_log_missing(1.2s3 epoch 29/28 rep_tid 25 entries 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 -- 0x558e846e2480 con 0x558e818d6800 2019-12-07T19:34:06.668+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_update_log_missing(1.2s4 epoch 29/28 rep_tid 25 entries 29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 -- 0x558e846e2000 con 0x558e818fe000 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e84558c00 seq 34 2019-12-07T19:34:06.668+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000029.00000000000000000013' <- 0000000029.00000000000000000013 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._fastinfo' <- _fastinfo 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84558c00 cost 4717 (1 ios * 4000 + 717 bytes) 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84558c00 onodes shared_blobs 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84558c00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558c00 prepare 2019-12-07T19:34:06.668+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84558c00 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558c00 io_done 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 lua=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] op_applied version 29'13 2019-12-07T19:34:06.668+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] calc_trim_to_aggressive limit = 27'12 2019-12-07T19:34:06.669+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e84176b00 finish 2019-12-07T19:34:06.669+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.669+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.669+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.669+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:06.669+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1163 < min_flush_size 524288 2019-12-07T19:34:06.669+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 3 2019-12-07T19:34:06.669+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2257 < min_flush_size 524288 2019-12-07T19:34:06.669+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 2 2019-12-07T19:34:06.669+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2824 < min_flush_size 524288 2019-12-07T19:34:06.669+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:06.669+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2847 < min_flush_size 524288 2019-12-07T19:34:06.669+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2847 < min_flush_size 524288 2019-12-07T19:34:06.669+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.669+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xe2ce9~b1f to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.669+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xe2ce9 0xe2ce9~b1f to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.669+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.669+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb2ce9 2019-12-07T19:34:06.669+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xce9 2019-12-07T19:34:06.669+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.669+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x808 and padding block with 0x7f8 2019-12-07T19:34:06.669+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 564443 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.670+0000 7fe2cea74700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:06.670+0000 7fe2cea74700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] SnapTrimmer state: NotTrimming: trimming 4 2019-12-07T19:34:06.670+0000 7fe2cea74700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim_wait trimq=[4~1] ps=[2~1]] exit Trimming/WaitReservation 2019-12-07T19:34:06.670+0000 7fe2cea74700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.670+0000 7fe2cea74700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:81 2019-12-07T19:34:06.670+0000 7fe2cea74700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:06.670+0000 7fe2cea74700 10 osd.0 29 queueing pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean trimq=[4~1] ps=[2~1]] for snaptrim 2019-12-07T19:34:06.670+0000 7fe2cea74700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=29) prio 5 cost 1048576 e29) 2019-12-07T19:34:06.670+0000 7fe2cea74700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.670+0000 7fe2cea74700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:82 2019-12-07T19:34:06.670+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 40 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558e83ebbc00 con 0x558e818d6000 2019-12-07T19:34:06.670+0000 7fe2e753b700 15 osd.0 29 enqueue_peering_evt 1.5s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:06.670+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.670+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xe3808 2019-12-07T19:34:06.670+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.670+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.670+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.670+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.670+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 3 cleaned 0 in 0.001918747s (0.000000091s flush + 0.001918656s kv commit) 2019-12-07T19:34:06.670+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.670+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.670+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e84558600,0x558e84558900,0x558e84558c00> 2019-12-07T19:34:06.670+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.670+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558600 kv_submitted 2019-12-07T19:34:06.670+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84558600 2019-12-07T19:34:06.670+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558600 finishing 2019-12-07T19:34:06.670+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84558600 onodes 2019-12-07T19:34:06.670+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84558600 done 2019-12-07T19:34:06.670+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84558900 kv_submitted 2019-12-07T19:34:06.670+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84558600 [] 2019-12-07T19:34:06.670+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.670+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558900 kv_submitted 2019-12-07T19:34:06.670+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84558900 2019-12-07T19:34:06.670+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558900 finishing 2019-12-07T19:34:06.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84558900 onodes 2019-12-07T19:34:06.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84558900 done 2019-12-07T19:34:06.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84558c00 kv_submitted 2019-12-07T19:34:06.671+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84558900 [] 2019-12-07T19:34:06.671+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.671+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558c00 kv_submitted 2019-12-07T19:34:06.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84558c00 2019-12-07T19:34:06.671+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558c00 finishing 2019-12-07T19:34:06.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84558c00 onodes 2019-12-07T19:34:06.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84558c00 done 2019-12-07T19:34:06.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:06.671+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84558c00 [] 2019-12-07T19:34:06.671+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.671+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=29) prio 5 cost 1048576 e29) queued 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=29) prio 5 cost 1048576 e29) pg 0x558e81075000 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] snap_trimmer posting 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 4 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 111 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1211+0+0 (unknown 0 0 0) 0x558e81229500 con 0x558e818fe400 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 1 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 15 osd.0 29 enqueue_peering_evt 1.bs1 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore.OmapIteratorImpl(0x558e81978540) upper_bound after SNA_1_0000000000000004_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.0_0000000000000001.2' 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore.OmapIteratorImpl(0x558e81978540) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.0_0000000000000001.28BF48E5.4.smithi13140796-39..' 2019-12-07T19:34:06.671+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 35 ==== pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 ==== 53+0+0 (unknown 0 0 0) 0x558e8118fa00 con 0x558e818fe000 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000004_.0_0000000000000001.2) returns 0 SNA_1_0000000000000004_.0_0000000000000001.28BF48E5.4.smithi13140796-39..,buffer::list(len=70, buffer::ptr(0~70 0x558e817e0c60 in raw 0x558e817e0c60 len 70 nref 1) ) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 snap_mapper.get_next_objects_to_trim SNA_1_0000000000000004_.0_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.671+0000 7fe2e753b700 15 osd.0 29 enqueue_op 0x558e83edcf20 prio 196 cost 0 latency 0.000017 epoch 29 pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 1 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 112 ==== pg_info2(1.3s5 1.3s5 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558e81229500 con 0x558e818fe400 2019-12-07T19:34:06.671+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore.OmapIteratorImpl(0x558e81978540) upper_bound after SNA_1_0000000000000004_.0_0000000000000001.28BF48E5.4.smithi13140796-39.. key 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.0_0000000000000001.28BF48E5.4.smithi13140796-39..' 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 15 osd.0 29 enqueue_peering_evt 1.3s5 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore.OmapIteratorImpl(0x558e81978540) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.1_0000000000000001.198F0B8C.4.smithi13140796-14..' 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000004_.0_0000000000000001.28BF48E5.4.smithi13140796-39..) returns 0 SNA_1_0000000000000004_.1_0000000000000001.198F0B8C.4.smithi13140796-14..,buffer::list(len=70, buffer::ptr(0~70 0x558e817e0c60 in raw 0x558e817e0c60 len 70 nref 1) ) 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 113 ==== pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 ==== 53+0+0 (unknown 0 0 0) 0x558e818a8a00 con 0x558e818fe400 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 15 osd.0 29 enqueue_op 0x558e84177ce0 prio 196 cost 0 latency 0.000011 epoch 29 pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 2019-12-07T19:34:06.671+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 41 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558e83ebbc00 con 0x558e818d6000 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] SnapTrimmer state: AwaitAsyncWork react trimming 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:06.671+0000 7fe2e753b700 15 osd.0 29 enqueue_peering_evt 1.5s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: obc NOT found in cache: 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:06.671+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# _ 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 114 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1211+0+0 (unknown 0 0 0) 0x558e81229500 con 0x558e818fe400 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# _ = 0 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 15 osd.0 29 enqueue_peering_evt 1.bs1 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.671+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 55 ==== pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 ==== 53+0+0 (unknown 0 0 0) 0x558e8118f800 con 0x558e818d6800 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] populate_obc_watchers 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:06.671+0000 7fe2e753b700 15 osd.0 29 enqueue_op 0x558e83edd080 prio 196 cost 0 latency 0.000009 epoch 29 pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 2019-12-07T19:34:06.671+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] PrimaryLogPG::check_blacklisted_obc_watchers for obc 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:06.671+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattrs 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattrs 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# = 0 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 115 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e81229500 con 0x558e818fe400 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: creating obc from disk: 0x558e84409680 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 15 osd.0 29 enqueue_peering_evt 1.bs1 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: 0x558e84409680 1:41df217a:::smithi13140796-39:4 rwstate(none n=0 w=0) oi: 1:41df217a:::smithi13140796-39:4(27'11 client.4540.0:116 dirty|data_digest s 638976 uv 5 dd b038798b alloc_hint [0 0 0]) exists: 1 ssc: 0x558e83dcc6c0 snapset: 4=[]:{4=[4]} 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 116 ==== osd pg remove(epoch 29; pg1.bs3; ) v3 ==== 32+0+0 (unknown 0 0 0) 0x558e819b1e00 con 0x558e818fe400 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: found obc in cache: 0x558e81222900 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 7 osd.0 29 handle_fast_pg_remove osd pg remove(epoch 29; pg1.bs3; ) v3 from osd.4 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 15 osd.0 29 enqueue_peering_evt 1.bs3 epoch_sent: 29 epoch_requested: 29 DeleteStart 2019-12-07T19:34:06.671+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 DeleteStart) prio 255 cost 10 e29) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_object_context: 0x558e81222900 1:41df217a:::smithi13140796-39:head rwstate(none n=0 w=0) oi: 1:41df217a:::smithi13140796-39:head(27'12 client.4540.0:231 whiteout|dirty s 0 uv 12 alloc_hint [0 0 0]) exists: 1 ssc: 0x558e83dcc6c0 snapset: 4=[]:{4=[4]} 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 1:41df217a:::smithi13140796-39:4 old_snaps 4 old snapset 4=[]:{4=[4]} 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] simple_opc_create 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 1:41df217a:::smithi13140796-39:4 snaps 4 -> ... deleting 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) pg 0x558e83c78000 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 1:41df217a:::smithi13140796-39:4 new snapset 4=[]:{} on 1:41df217a:::smithi13140796-39:head(27'12 client.4540.0:231 whiteout|dirty s 0 uv 12 alloc_hint [0 0 0]) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 1:41df217a:::smithi13140796-39:4 removing 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.671+0000 7fe2c6a64700 -1 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] removing snap head 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] trim_object trimming whiteout on 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e84559b00 seq 11 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] new_repop rep_tid 26 (no op) 2019-12-07T19:34:06.671+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] new_repop: repgather(0x558e83dccb40 0'0 rep_tid=26 committed?=0 r=0) 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84559b00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] simple_opc_submit 0x558e83dccb40 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84559b00 onodes shared_blobs 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84559b00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.671+0000 7fe2c6a64700 7 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] issue_repop rep_tid 26 o 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559b00 prepare 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84559b00 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559b00 io_done 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] submit_transaction: op Op(1:41df217a:::smithi13140796-39:4 v=29'15 tt=0'0 tid=26 reqid=osd.0.0:26 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) starting 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: Getting attr on 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: not in cache 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) pg 0x558e81231000 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: found on disk, size 159744 2019-12-07T19:34:06.671+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# hinfo_key 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# hinfo_key = 0 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): delete, setting projected size to 0 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): 1:41df217a:::smithi13140796-39:4 projected size 0 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: Getting attr on 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: not in cache 1:41df217a:::smithi13140796-39:head 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e845c2000 seq 11 2019-12-07T19:34:06.671+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] get_hash_info: found on disk, size 0 2019-12-07T19:34:06.671+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# hinfo_key = 0 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845c2000 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845c2000 onodes shared_blobs 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845c2000 allocated 0x[] released 0x[] 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): delete, setting projected size to 0 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2000 prepare 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845c2000 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2000 io_done 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): 1:41df217a:::smithi13140796-39:head projected size 0 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) pg 0x558e83c78000 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] start_rmw: Op(1:41df217a:::smithi13140796-39:4 v=29'15 tt=0'0 tid=26 reqid=osd.0.0:26 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:4=[],1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_state_to_reads: Op(1:41df217a:::smithi13140796-39:4 v=29'15 tt=0'0 tid=26 reqid=osd.0.0:26 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:4=[],1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: starting commit on Op(1:41df217a:::smithi13140796-39:4 v=29'15 tt=0'0 tid=26 reqid=osd.0.0:26 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:4=[],1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e845c2300 seq 12 2019-12-07T19:34:06.671+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._biginfo' <- _biginfo 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] generate_transactions: 1:41df217a:::smithi13140796-39:4, current size is 638976 buffers are {} 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845c2300 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845c2300 onodes shared_blobs 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845c2300 allocated 0x[] released 0x[] 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2300 prepare 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845c2300 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2300 io_done 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e29) pg 0x558e83c78000 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): new_size start 0 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): to_overwrite: {} 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): to_append: {} 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): 1:41df217a:::smithi13140796-39:4 resetting hinfo to logical size 0 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e845c2600 seq 13 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] generate_transactions: 1:41df217a:::smithi13140796-39:head, current size is 0 buffers are {} 2019-12-07T19:34:06.671+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._biginfo' <- _biginfo 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845c2600 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845c2600 onodes shared_blobs 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845c2600 allocated 0x[] released 0x[] 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2600 prepare 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845c2600 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2600 io_done 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): new_size start 0 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 DeleteStart) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): to_overwrite: {} 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 DeleteStart) prio 255 cost 10 e29) pg 0x558e81928000 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): to_append: {} 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] operator(): 1:41df217a:::smithi13140796-39:head resetting hinfo to logical size 0 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 DeleteStart 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:06.671+0000 7fe2ca26b700 5 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] exit Started/Stray 2019-12-07T19:34:06.672271+0000 6 0.000092 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: written: {1:41df217a:::smithi13140796-39:4={},1:41df217a:::smithi13140796-39:head={}} 2019-12-07T19:34:06.671+0000 7fe2ca26b700 5 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] enter Started/ToDelete 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: op: Op(1:41df217a:::smithi13140796-39:4 v=29'15 tt=0'0 tid=26 reqid=osd.0.0:26 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:4=[],1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:34:06.671+0000 7fe2ca26b700 5 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] enter Started/ToDelete/WaitDeleteReseved 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: written_set: {1:41df217a:::smithi13140796-39:4=[],1:41df217a:::smithi13140796-39:head=[]} 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: 1:41df217a:::smithi13140796-39:4,{} 2019-12-07T19:34:06.671+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.671+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: 1:41df217a:::smithi13140796-39:head,{} 2019-12-07T19:34:06.671+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 29/28 ECSubWrite(tid=26, reqid=osd.0.0:26, at_version=29'15, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e819da800 con 0x558e818d7c00 2019-12-07T19:34:06.671+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 29/28 ECSubWrite(tid=26, reqid=osd.0.0:26, at_version=29'15, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e83e56000 con 0x558e818d7000 2019-12-07T19:34:06.671+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 29/28 ECSubWrite(tid=26, reqid=osd.0.0:26, at_version=29'15, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e83e55000 con 0x558e818fe400 2019-12-07T19:34:06.671+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s3 29/28 ECSubWrite(tid=26, reqid=osd.0.0:26, at_version=29'15, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e811b8800 con 0x558e818d6800 2019-12-07T19:34:06.671+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=26, reqid=osd.0.0:26, at_version=29'15, trim_to=0'0, roll_forward_to=0'0)) v2 -- 0x558e811b8000 con 0x558e818fe000 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 snap_mapper.remove_oid 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 snap_mapper._remove_oid 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:06.671+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.671+0000 7fe2cea74700 15 osd.0 29 enqueue_peering_evt 1.bs3 epoch_sent: 29 epoch_requested: 29 DeleteReserved 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:06.671+0000 7fe2cea74700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 DeleteReserved) prio 255 cost 10 e29) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.671+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 snap_mapper.get_snaps 1:41df217a:::smithi13140796-39:4 4 2019-12-07T19:34:06.671+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 DeleteReserved) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 snap_mapper.clear_snaps 1:41df217a:::smithi13140796-39:4 2019-12-07T19:34:06.671+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 snap_mapper.clear_snaps rm OBJ_.0_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:06.671+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 DeleteReserved) prio 255 cost 10 e29) pg 0x558e81928000 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 snap_mapper._remove_oid rm SNA_1_0000000000000004_.0_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 earliest_dup_version = 0 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 DeleteReserved 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 trim 29'14 (27'11) delete 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 trim 29'15 (27'12) delete 1:41df217a:::smithi13140796-39:head by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.671+0000 7fe2c6263700 5 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] exit Started/ToDelete/WaitDeleteReseved 2019-12-07T19:34:06.672464+0000 1 0.000041 2019-12-07T19:34:06.671+0000 7fe2c6263700 5 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] enter Started/ToDelete/Deleting 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] append_log log((0'0,29'13], crt=27'12) [29'14 (27'11) delete 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,29'15 (27'12) delete 1:41df217a:::smithi13140796-39:head by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] on_removal 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] on_shutdown 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'14 (0'0,29'14] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 lua=29'13 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] add_log_entry 29'14 (27'11) delete 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] cancel_copy_ops 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] cancel_flush_ops 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 lua=29'13 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] add_log_entry 29'15 (27'12) delete 1:41df217a:::smithi13140796-39:head by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] cancel_proxy_ops 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 lua=29'13 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] append_log approx pg log length = 15 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] cancel_manifest_ops 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 lua=29'13 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.671+0000 7fe2c6a64700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'14, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] clear_backoffs 2019-12-07T19:34:06.671+0000 7fe2c6263700 20 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] exit NotTrimming 2019-12-07T19:34:06.671+0000 7fe2c6263700 20 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] enter NotTrimming 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] on_change 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] clear_async_reads 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e83e73500 seq 35 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] release_backoffs [1:d0000000::::0,1:e0000000::::head) 2019-12-07T19:34:06.671+0000 7fe2c6263700 20 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] agent_stop 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:41df217a:::smithi13140796-39:4#e key 0x80800000000000000141df21'z!smithi13140796-39!='0x0000000000000004000000000000000e'o' 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] cancel_recovery 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] clear_recovery_state 2019-12-07T19:34:06.671+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# -> 0#1:41df217a:::smithi13140796-39:4#e 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.2s0_head 0#1:41df217a:::smithi13140796-39:4# -> 0#1:41df217a:::smithi13140796-39:4#e = 0 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) get_onode oid 0#1:41df217a:::smithi13140796-39:head#f key 0x80800000000000000141df21'z!smithi13140796-39!='0xfffffffffffffffe000000000000000f'o' 2019-12-07T19:34:06.671+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e83c57800 seq 30 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.2s0_head 0x558e819b12c0) r -2 v.len 0 2019-12-07T19:34:06.671+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# -> 0#1:41df217a:::smithi13140796-39:head#f 2019-12-07T19:34:06.671+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# 2019-12-07T19:34:06.671+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._epoch' <- _epoch 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.2s0_head 0#1:41df217a:::smithi13140796-39:head# -> 0#1:41df217a:::smithi13140796-39:head#f = 0 2019-12-07T19:34:06.671+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042e'._info' <- _info 2019-12-07T19:34:06.671+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c57800 cost 5210 (1 ios * 4000 + 1210 bytes) 2019-12-07T19:34:06.671+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c57800 onodes shared_blobs 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.0_0000000000000001.28BF48E5.4.smithi13140796-39..' <- OBJ_.0_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:06.671+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c57800 allocated 0x[] released 0x[] 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.671+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.0_0000000000000001.28BF48E5.4.smithi13140796-39..' <- SNA_1_0000000000000004_.0_0000000000000001.28BF48E5.4.smithi13140796-39.. 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 prepare 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.671+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c57800 2019-12-07T19:34:06.671+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 io_done 2019-12-07T19:34:06.671+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:06.671+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000029.00000000000000000014' <- 0000000029.00000000000000000014 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.0000000029.00000000000000000015' <- 0000000029.00000000000000000015 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:06.671+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e73500 cost 6674 (1 ios * 4000 + 2674 bytes) 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e73500 onodes 0x558e81a03680,0x558e83cb3680 shared_blobs 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e81a03788) update 0#1:41df217a:::smithi13140796-39:4#e 2019-12-07T19:34:06.671+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:41df217a:::smithi13140796-39:4#e is 958 (412 bytes onode + 2 bytes spanning blobs + 544 bytes inline extents) 2019-12-07T19:34:06.671+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 5 submitting 5 deferred done 0 stable 0 2019-12-07T19:34:06.671+0000 7fe2c6a64700 20 bluestore.extentmap(0x558e83cb3788) update 0#1:41df217a:::smithi13140796-39:head#f 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 0#1:41df217a:::smithi13140796-39:head#f is 475 (467 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e73500 allocated 0x[] released 0x[] 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73500 prepare 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e73500 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73500 io_done 2019-12-07T19:34:06.672+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1023 < min_flush_size 524288 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 lua=29'13 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] op_applied version 29'15 2019-12-07T19:34:06.672+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 3 2019-12-07T19:34:06.672+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2046 < min_flush_size 524288 2019-12-07T19:34:06.672+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] eval_repop repgather(0x558e83dccb40 29'15 rep_tid=26 committed?=0 r=0) (no op) 2019-12-07T19:34:06.672+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3140 < min_flush_size 524288 2019-12-07T19:34:06.672+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 2 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] calc_trim_to_aggressive limit = 27'12 2019-12-07T19:34:06.672+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4250 < min_flush_size 524288 2019-12-07T19:34:06.672+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:06.672+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5297 < min_flush_size 524288 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:06.672+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:34:06.672+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5320 < min_flush_size 524288 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] enter Trimming/WaitRepops 2019-12-07T19:34:06.672+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5320 < min_flush_size 524288 2019-12-07T19:34:06.672+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] snap_trimmer complete 2019-12-07T19:34:06.672+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xe3808~14c8 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.672+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xe3808 0xe3808~14c8 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.672+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.672+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb3808 2019-12-07T19:34:06.672+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x808 2019-12-07T19:34:06.672+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.672+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xcd0 and padding block with 0x330 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 osd.0 29 dequeue_op 0x558e83edcf20 prio 196 cost 0 latency 0.000998 pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_update_log_missing_reply got reply from 7(4) 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 osd.0 29 dequeue_op 0x558e83edcf20 finish 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 osd.0 29 dequeue_op 0x558e84177ce0 prio 196 cost 0 latency 0.000993 pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_update_log_missing_reply got reply from 4(1) 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 osd.0 29 dequeue_op 0x558e84177ce0 finish 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 osd.0 29 dequeue_op 0x558e83edd080 prio 196 cost 0 latency 0.000983 pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_update_log_missing_reply got reply from 5(3) 2019-12-07T19:34:06.672+0000 7fe2c6a64700 10 osd.0 29 dequeue_op 0x558e83edd080 finish 2019-12-07T19:34:06.672+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e29) pg 0x558e83d07000 2019-12-07T19:34:06.672+0000 7fe2c9269700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944908336s) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active pruub 44.064407826s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active pruub 44.064407826s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.672+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:06.672+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e845c3b00 seq 13 2019-12-07T19:34:06.672+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:06.672+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:06.672+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845c3b00 cost 5362 (1 ios * 4000 + 1362 bytes) 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845c3b00 onodes shared_blobs 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845c3b00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.672+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c3b00 prepare 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845c3b00 2019-12-07T19:34:06.672+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c3b00 io_done 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e29) pg 0x558e83d07000 2019-12-07T19:34:06.672+0000 7fe2c9269700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.672+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:06.672+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e845d4000 seq 14 2019-12-07T19:34:06.672+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._biginfo' <- _biginfo 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:06.672+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:06.672+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845d4000 cost 5427 (1 ios * 4000 + 1427 bytes) 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845d4000 onodes shared_blobs 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845d4000 allocated 0x[] released 0x[] 2019-12-07T19:34:06.672+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4000 prepare 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845d4000 2019-12-07T19:34:06.672+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4000 io_done 2019-12-07T19:34:06.672+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 197 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558e8404bc00 con 0x558e818d7c00 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.es4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 198 ==== pg_info2(1.9s3 1.9s3 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558e83c9c700 con 0x558e818d7c00 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.9s3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 199 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558e83c9c700 con 0x558e818d7c00 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.8s5 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 200 ==== pg_info2(1.7s3 1.7s3 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1211+0+0 (unknown 0 0 0) 0x558e83c9c700 con 0x558e818d7c00 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.7s3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 201 ==== pg_info2(1.7s3 1.7s3 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1211+0+0 (unknown 0 0 0) 0x558e83c9c700 con 0x558e818d7c00 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.7s3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 202 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558e83c9c700 con 0x558e818d7c00 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.es4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.672+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 203 ==== pg_info2(1.9s3 1.9s3 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558e83c9c700 con 0x558e818d7c00 2019-12-07T19:34:06.673+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.9s3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:06.673+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.673+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 204 ==== pg_info2(1.7s3 1.7s3 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c9c700 con 0x558e818d7c00 2019-12-07T19:34:06.673+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.7s3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:06.673+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.673+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 205 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c9c700 con 0x558e818d7c00 2019-12-07T19:34:06.673+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.es4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:06.673+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) pg 0x558e8192b000 2019-12-07T19:34:06.673+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.941957470s) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active pruub 44.060646370s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active pruub 44.060646370s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.673+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:06.673+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83d4c600 seq 24 2019-12-07T19:34:06.673+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:06.673+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:06.673+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c600 cost 5310 (1 ios * 4000 + 1310 bytes) 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c600 onodes shared_blobs 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c600 allocated 0x[] released 0x[] 2019-12-07T19:34:06.673+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c600 prepare 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c600 2019-12-07T19:34:06.673+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c600 io_done 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) pg 0x558e8192b000 2019-12-07T19:34:06.673+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.673+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:06.673+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83d4c000 seq 25 2019-12-07T19:34:06.673+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:06.673+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:06.673+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c000 cost 5375 (1 ios * 4000 + 1375 bytes) 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c000 onodes shared_blobs 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c000 allocated 0x[] released 0x[] 2019-12-07T19:34:06.673+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 prepare 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c000 2019-12-07T19:34:06.673+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 io_done 2019-12-07T19:34:06.673+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.673+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.673+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.673+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.673+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e29) pg 0x558e841a4000 2019-12-07T19:34:06.673+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.946615610s) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'23 active pruub 44.065109905s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:06.673+0000 7fe2cba6e700 20 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'23 active pruub 44.065109905s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.673+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:06.673+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:06.673+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e81a0a300 seq 25 2019-12-07T19:34:06.673+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:06.673+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:06.673+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:06.673+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a300 cost 5310 (1 ios * 4000 + 1310 bytes) 2019-12-07T19:34:06.673+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a300 onodes shared_blobs 2019-12-07T19:34:06.673+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a300 allocated 0x[] released 0x[] 2019-12-07T19:34:06.673+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 prepare 2019-12-07T19:34:06.673+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a300 2019-12-07T19:34:06.673+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 io_done 2019-12-07T19:34:06.673+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) pg 0x558e8122e000 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942150657s) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active pruub 44.061193946s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active pruub 44.061193946s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e8195f500 seq 32 2019-12-07T19:34:06.673+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195f500 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195f500 onodes shared_blobs 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195f500 allocated 0x[] released 0x[] 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 prepare 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195f500 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 io_done 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) pg 0x558e8122e000 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e81a0b500 seq 33 2019-12-07T19:34:06.673+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b500 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b500 onodes shared_blobs 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b500 allocated 0x[] released 0x[] 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 prepare 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b500 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 io_done 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e29) pg 0x558e8122e000 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e81a0a900 seq 34 2019-12-07T19:34:06.673+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a900 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a900 onodes shared_blobs 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a900 allocated 0x[] released 0x[] 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 prepare 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a900 2019-12-07T19:34:06.673+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 io_done 2019-12-07T19:34:06.673+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.674+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 206 ==== MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=135, reqid=client.4540.0:244, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 190744+0+0 (unknown 0 0 0) 0x558e819db000 con 0x558e818d7c00 2019-12-07T19:34:06.674+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e84176840 prio 127 cost 0 latency 0.000034 epoch 29 MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=135, reqid=client.4540.0:244, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.674+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=135, reqid=client.4540.0:244, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.674+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 36 ==== MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e8118fc00 con 0x558e818fe000 2019-12-07T19:34:06.674+0000 7fe2e753b700 15 osd.0 29 enqueue_op 0x558e83edc2c0 prio 196 cost 0 latency 0.000015 epoch 29 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.674+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=135, reqid=client.4540.0:244, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=135, reqid=client.4540.0:244, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558e8192b000 2019-12-07T19:34:06.674+0000 7fe2c7265700 10 osd.0 29 dequeue_op 0x558e84176840 prio 127 cost 0 latency 0.000345 MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=135, reqid=client.4540.0:244, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.674+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=135, reqid=client.4540.0:244, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.674+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 27'8 (0'0,27'8] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] append_log log((0'0,27'8], crt=27'8) [29'9 (21'6) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:244 2019-12-07T19:34:05.664687+0000 0 ObjectCleanRegions clean_offsets: [0~212992,966656~18446744073708584959], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.674+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'9 (0'0,29'9] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=27'8 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] add_log_entry 29'9 (21'6) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:244 2019-12-07T19:34:05.664687+0000 0 ObjectCleanRegions clean_offsets: [0~212992,966656~18446744073708584959], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.674+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'9 (0'0,29'9] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=27'8 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] append_log approx pg log length = 9 2019-12-07T19:34:06.674+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'9 (0'0,29'9] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=27'8 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.674+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.674+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 207 ==== pg_info2(1.as3 1.as3 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1211+0+0 (unknown 0 0 0) 0x558e83c48a00 con 0x558e818d7c00 2019-12-07T19:34:06.674+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'9, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.674+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.as3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:06.674+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.674+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e819a0000 seq 26 2019-12-07T19:34:06.674+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 2 keys 2019-12-07T19:34:06.674+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 2 keys = 0 2019-12-07T19:34:06.674+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:06.674+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:06.674+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xd000~2e000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:9da56077:::smithi13140796-13:head# 0xd000~2e000 - have 0xd000 (53248) bytes fadvise_flags 0x0 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:06.674+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0xd000~2e000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a0000 23 blobs 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.674+0000 7fe2c7265700 10 fbmap_alloc 0x558e80233300 allocate 0x2e000/1000,2e000,0 2019-12-07T19:34:06.674+0000 7fe2c7265700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x586000~2e000/1000,2e000,0 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x586000~2e000] 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a24d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84030c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a24d0 blob([0x586000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84030c40 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e841a24d0) get_ref 0x0~2000 Blob(0x558e841a24d0 blob([0x586000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84030c40 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e841a24d0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xd000~2000: 0x0~2000 Blob(0x558e841a24d0 blob([0x586000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84030c40 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84030c58 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e818660e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b282a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e818660e0 blob([0x588000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b282a0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e818660e0) get_ref 0x0~2000 Blob(0x558e818660e0 blob([0x588000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b282a0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e818660e0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xf000~2000: 0x0~2000 Blob(0x558e818660e0 blob([0x588000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b282a0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83b282b8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a2310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8423c5b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a2310 blob([0x58a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423c5b0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e841a2310) get_ref 0x0~2000 Blob(0x558e841a2310 blob([0x58a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423c5b0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e841a2310) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x11000~2000: 0x0~2000 Blob(0x558e841a2310 blob([0x58a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423c5b0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8423c5c8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841db490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841db500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e29) pg 0x558e8193b000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841db490 blob([0x58c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db500 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e841db490) get_ref 0x0~2000 Blob(0x558e841db490 blob([0x58c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db500 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e841db490) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x13000~2000: 0x0~2000 Blob(0x558e841db490 blob([0x58c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db500 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841db518 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841db570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841db5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841db570 blob([0x58e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db5e0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.944939721s) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'2 active pruub 44.064869888s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e841db570) get_ref 0x0~2000 Blob(0x558e841db570 blob([0x58e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db5e0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e841db570) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x15000~2000: 0x0~2000 Blob(0x558e841db570 blob([0x58e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db5e0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841db5f8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841db650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841db6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'2 active pruub 44.064869888s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841db650 blob([0x590000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db6c0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e841db650) get_ref 0x0~2000 Blob(0x558e841db650 blob([0x590000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db6c0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e841db650) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x17000~2000: 0x0~2000 Blob(0x558e841db650 blob([0x590000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db6c0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e841db6d8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e84558f00 seq 18 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841db730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472e000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841db730 blob([0x592000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e000 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e841db730) get_ref 0x0~2000 Blob(0x558e841db730 blob([0x592000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e000 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e841db730) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x19000~2000: 0x0~2000 Blob(0x558e841db730 blob([0x592000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472e000 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472e018 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84558f00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84558f00 onodes shared_blobs 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84558f00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472e070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472e0e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472e070 blob([0x594000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e0e0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e070) get_ref 0x0~2000 Blob(0x558e8472e070 blob([0x594000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e0e0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e070) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1b000~2000: 0x0~2000 Blob(0x558e8472e070 blob([0x594000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472e0e0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472e0f8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558f00 prepare 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84558f00 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558f00 io_done 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472e150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472e1c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472e150 blob([0x596000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e1c0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e150) get_ref 0x0~2000 Blob(0x558e8472e150 blob([0x596000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e1c0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e150) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1d000~2000: 0x0~2000 Blob(0x558e8472e150 blob([0x596000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472e1c0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472e1d8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _finish_recovery 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472e230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472e2a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472e230 blob([0x598000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e2a0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e230) get_ref 0x0~2000 Blob(0x558e8472e230 blob([0x598000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e2a0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e230) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1f000~2000: 0x0~2000 Blob(0x558e8472e230 blob([0x598000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472e2a0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472e2b8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472e310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472e380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] purge_strays 5(4) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472e310 blob([0x59a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e380 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e310) get_ref 0x0~2000 Blob(0x558e8472e310 blob([0x59a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e380 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e310) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x21000~2000: 0x0~2000 Blob(0x558e8472e310 blob([0x59a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472e380 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472e398 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472e3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472e460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472e3f0 blob([0x59c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e460 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e3f0) get_ref 0x0~2000 Blob(0x558e8472e3f0 blob([0x59c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e460 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e3f0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x23000~2000: 0x0~2000 Blob(0x558e8472e3f0 blob([0x59c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472e460 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472e478 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] sending PGRemove to osd.5(4) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472e4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472e540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472e4d0 blob([0x59e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e540 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e4d0) get_ref 0x0~2000 Blob(0x558e8472e4d0 blob([0x59e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e540 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x25000~2000: 0x0~2000 Blob(0x558e8472e4d0 blob([0x59e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472e540 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472e558 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- osd pg remove(epoch 29; pg1.2s4; ) v3 -- 0x558e8122c1e0 con 0x558e818d6800 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472e5b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472e620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472e5b0 blob([0x5a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e620 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e5b0) get_ref 0x0~2000 Blob(0x558e8472e5b0 blob([0x5a0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e620 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e5b0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x27000~2000: 0x0~2000 Blob(0x558e8472e5b0 blob([0x5a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472e620 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472e638 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472e690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472e700 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472e690 blob([0x5a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e700 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e690) get_ref 0x0~2000 Blob(0x558e8472e690 blob([0x5a2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e700 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e690) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x29000~2000: 0x0~2000 Blob(0x558e8472e690 blob([0x5a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472e700 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] remove_stray_recovery_sources remove osd 5(4) from missing_loc 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472e718 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472e770 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472e7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472e770 blob([0x5a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e7e0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e770) get_ref 0x0~2000 Blob(0x558e8472e770 blob([0x5a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e7e0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e770) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] update_heartbeat_peers 0,1,3,4,5,7 unchanged 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2b000~2000: 0x0~2000 Blob(0x558e8472e770 blob([0x5a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472e7e0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472e7f8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472e850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472e8c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472e850 blob([0x5a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e8c0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e850) get_ref 0x0~2000 Blob(0x558e8472e850 blob([0x5a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e8c0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e850) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2d000~2000: 0x0~2000 Blob(0x558e8472e850 blob([0x5a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472e8c0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472e8d8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:83 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472e930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472e9a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472e930 blob([0x5a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e9a0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e930) get_ref 0x0~2000 Blob(0x558e8472e930 blob([0x5a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472e9a0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472e930) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2f000~2000: 0x0~2000 Blob(0x558e8472e930 blob([0x5a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472e9a0 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472e9b8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472ea10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472ea80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472ea10 blob([0x5aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472ea80 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472ea10) get_ref 0x0~2000 Blob(0x558e8472ea10 blob([0x5aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472ea80 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472ea10) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x31000~2000: 0x0~2000 Blob(0x558e8472ea10 blob([0x5aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472ea80 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472ea98 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e83edc2c0 prio 196 cost 0 latency 0.000672 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472eaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472eb60 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472eaf0 blob([0x5ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472eb60 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472eaf0) get_ref 0x0~2000 Blob(0x558e8472eaf0 blob([0x5ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472eb60 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472eaf0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x33000~2000: 0x0~2000 Blob(0x558e8472eaf0 blob([0x5ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472eb60 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472eb78 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e83edc2c0 finish 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472ebd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472ec40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472ebd0 blob([0x5ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472ec40 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472ebd0) get_ref 0x0~2000 Blob(0x558e8472ebd0 blob([0x5ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472ec40 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472ebd0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x35000~2000: 0x0~2000 Blob(0x558e8472ebd0 blob([0x5ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472ec40 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472ec58 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472ecb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472ed20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472ecb0 blob([0x5b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472ed20 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472ecb0) get_ref 0x0~2000 Blob(0x558e8472ecb0 blob([0x5b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472ed20 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.blob(0x558e8472ecb0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x37000~2000: 0x0~2000 Blob(0x558e8472ecb0 blob([0x5b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472ed20 sbid 0x0)) 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472ed38 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.674+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472ed90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472ee00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472ed90 blob([0x5b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472ee00 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore.blob(0x558e8472ed90) get_ref 0x0~2000 Blob(0x558e8472ed90 blob([0x5b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472ee00 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore.blob(0x558e8472ed90) get_ref init 0x2000, 1000 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x39000~2000: 0x0~2000 Blob(0x558e8472ed90 blob([0x5b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472ee00 sbid 0x0)) 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472ee18 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x3b000 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811ddc88) dirty_range mark inline shard dirty 2019-12-07T19:34:06.675+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0xd000~2e000 = 0 2019-12-07T19:34:06.675+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.675+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.675+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000029.00000000000000000009' <- 0000000029.00000000000000000009 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.675+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:06.675+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0000 cost 287184 (24 ios * 4000 + 191184 bytes) 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0000 onodes 0x558e811ddb80 shared_blobs 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811ddc88) update 3#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811ddc88) update inline shard 571 bytes from 30 extents 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9da56077:::smithi13140796-13:head# is 1062 (485 bytes onode + 2 bytes spanning blobs + 575 bytes inline extents) 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0000 allocated 0x[586000~2e000] released 0x[] 2019-12-07T19:34:06.675+0000 7fe2c7265700 10 freelist allocate 0x586000~2e000 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 freelist _xor first_key 0x580000 last_key 0x580000 2019-12-07T19:34:06.675+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 prepare 2019-12-07T19:34:06.675+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e819a0000 2019-12-07T19:34:06.675+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 56 ==== MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84053a00 con 0x558e818d6800 2019-12-07T19:34:06.675+0000 7fe2e753b700 15 osd.0 29 enqueue_op 0x558e841774a0 prio 196 cost 0 latency 0.000017 epoch 29 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.675+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.675+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e841774a0 prio 196 cost 0 latency 0.000045 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.675+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.675+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e841774a0 finish 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.675+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'9 (0'0,29'9] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=27'8 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] op_applied version 29'9 2019-12-07T19:34:06.675+0000 7fe2c7265700 10 osd.0 29 dequeue_op 0x558e84176840 finish 2019-12-07T19:34:06.675+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.675+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.675+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 aio_wait 2019-12-07T19:34:06.675+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0000 2019-12-07T19:34:06.675+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 io_done 2019-12-07T19:34:06.675+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 208 ==== pg_info2(1.as3 1.as3 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1211+0+0 (unknown 0 0 0) 0x558e83c48a00 con 0x558e818d7c00 2019-12-07T19:34:06.675+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.as3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:06.675+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e29) pg 0x558e8193b000 2019-12-07T19:34:06.675+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.675+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:06.675+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e84559200 seq 19 2019-12-07T19:34:06.675+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:06.675+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:06.675+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84559200 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84559200 onodes shared_blobs 2019-12-07T19:34:06.675+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84559200 allocated 0x[] released 0x[] 2019-12-07T19:34:06.676+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559200 prepare 2019-12-07T19:34:06.676+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84559200 2019-12-07T19:34:06.676+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559200 io_done 2019-12-07T19:34:06.676+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 111 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558e83c48a00 con 0x558e818d7000 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.1s1 epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e29) pg 0x558e819e4000 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 112 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558e8122b100 con 0x558e818d7000 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940748925s) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'7 active pruub 44.059111212s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.1s1 epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'7 active pruub 44.059111212s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 113 ==== pg_info2(1.fs1 1.fs1 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558e8122b100 con 0x558e818d7000 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e83d4cf00 seq 24 2019-12-07T19:34:06.676+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.fs1 epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4cf00 cost 5310 (1 ios * 4000 + 1310 bytes) 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4cf00 onodes shared_blobs 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4cf00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 prepare 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4cf00 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 io_done 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 114 ==== pg_info2(1.fs1 1.fs1 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558e8122b100 con 0x558e818d7000 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e29) pg 0x558e819e4000 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.fs1 epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'7 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'7 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'7 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e8417b800 seq 25 2019-12-07T19:34:06.676+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 115 ==== MOSDECSubOpWrite(1.1s1 29/28 ECSubWrite(tid=86, reqid=osd.3.0:86, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2656+0+0 (unknown 0 0 0) 0x558e83e56000 con 0x558e818d7000 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417b800 cost 5375 (1 ios * 4000 + 1375 bytes) 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417b800 onodes shared_blobs 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e84176580 prio 127 cost 0 latency 0.000024 epoch 29 MOSDECSubOpWrite(1.1s1 29/28 ECSubWrite(tid=86, reqid=osd.3.0:86, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417b800 allocated 0x[] released 0x[] 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 29/28 ECSubWrite(tid=86, reqid=osd.3.0:86, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 prepare 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417b800 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 io_done 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 29/28 ECSubWrite(tid=86, reqid=osd.3.0:86, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 29/28 ECSubWrite(tid=86, reqid=osd.3.0:86, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558e819e4000 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 osd.0 29 dequeue_op 0x558e84176580 prio 127 cost 0 latency 0.000050 MOSDECSubOpWrite(1.1s1 29/28 ECSubWrite(tid=86, reqid=osd.3.0:86, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=27'7 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.676+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=27'7 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.1s1 29/28 ECSubWrite(tid=86, reqid=osd.3.0:86, at_version=29'9, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 snap_mapper.remove_oid 1:891f0d13:::smithi13140796-14:4 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 snap_mapper._remove_oid 1:891f0d13:::smithi13140796-14:4 2019-12-07T19:34:06.676+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.676+0000 7fe2cb26d700 20 bluestore.onode(0x558e8125c280).flush flush cnt:1 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e29) pg 0x558e81a9b000 2019-12-07T19:34:06.676+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.940456110s) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active pruub 44.059243341s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active pruub 44.059243341s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.676+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:06.676+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e81a0b800 seq 30 2019-12-07T19:34:06.676+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:06.676+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:06.676+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b800 cost 5310 (1 ios * 4000 + 1310 bytes) 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b800 onodes shared_blobs 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b800 allocated 0x[] released 0x[] 2019-12-07T19:34:06.676+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 prepare 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b800 2019-12-07T19:34:06.676+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 io_done 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e29) pg 0x558e81a9b000 2019-12-07T19:34:06.676+0000 7fe2c425f700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.676+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:06.676+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e811fa300 seq 31 2019-12-07T19:34:06.676+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:06.676+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:06.676+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa300 cost 5375 (1 ios * 4000 + 1375 bytes) 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa300 onodes shared_blobs 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa300 allocated 0x[] released 0x[] 2019-12-07T19:34:06.676+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 prepare 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa300 2019-12-07T19:34:06.676+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 io_done 2019-12-07T19:34:06.676+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.676+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xe4cd0 2019-12-07T19:34:06.676+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.676+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.676+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000001 2019-12-07T19:34:06.676+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.676+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 117 ==== pg_info2(1.3s5 1.3s5 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558e81229500 con 0x558e818fe400 2019-12-07T19:34:06.676+0000 7fe2e6d3a700 15 osd.0 29 enqueue_peering_evt 1.3s5 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:06.676+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 209 ==== MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=136, reqid=client.4540.0:245, at_version=29'10, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 96536+0+0 (unknown 0 0 0) 0x558e819da800 con 0x558e818d7c00 2019-12-07T19:34:06.676+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 118 ==== pg_info2(1.3s5 1.3s5 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e81229500 con 0x558e818fe400 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e84177600 prio 127 cost 0 latency 0.000027 epoch 29 MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=136, reqid=client.4540.0:245, at_version=29'10, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.676+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=136, reqid=client.4540.0:245, at_version=29'10, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.676+0000 7fe2e6d3a700 15 osd.0 29 enqueue_peering_evt 1.3s5 epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:06.676+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.676+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 119 ==== MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818efc00 con 0x558e818fe400 2019-12-07T19:34:06.676+0000 7fe2e6d3a700 15 osd.0 29 enqueue_op 0x558e818051e0 prio 196 cost 0 latency 0.000013 epoch 29 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.676+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=136, reqid=client.4540.0:245, at_version=29'10, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=136, reqid=client.4540.0:245, at_version=29'10, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558e8192b000 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 osd.0 29 dequeue_op 0x558e84177600 prio 127 cost 0 latency 0.000113 MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=136, reqid=client.4540.0:245, at_version=29'10, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s3( v 29'9 (0'0,29'9] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'9 (0'0,29'9] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=136, reqid=client.4540.0:245, at_version=29'10, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) pg 0x558e81231000 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'9 (0'0,29'9] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] append_log log((0'0,29'9], crt=27'8) [29'10 (29'9) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:245 2019-12-07T19:34:05.667234+0000 0 ObjectCleanRegions clean_offsets: [0~966656,1343488~18446744073708208127], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'10 (0'0,29'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'9 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] add_log_entry 29'10 (29'9) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:245 2019-12-07T19:34:05.667234+0000 0 ObjectCleanRegions clean_offsets: [0~966656,1343488~18446744073708208127], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'10 (0'0,29'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'9 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] append_log approx pg log length = 10 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'10 (0'0,29'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'9 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.677+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'10, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e819a1200 seq 27 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.677+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 2 keys 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 2 keys = 0 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e845c2900 seq 12 2019-12-07T19:34:06.677+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:06.677+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:06.677+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3b000~17000 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._biginfo' <- _biginfo 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:9da56077:::smithi13140796-13:head# 0x3b000~17000 - have 0x3b000 (241664) bytes fadvise_flags 0x0 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x3b000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845c2900 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845c2900 onodes shared_blobs 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845c2900 allocated 0x[] released 0x[] 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a1200 12 blobs 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2900 prepare 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845c2900 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2900 io_done 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e29) pg 0x558e81231000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e845c2c00 seq 13 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.677+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._biginfo' <- _biginfo 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 fbmap_alloc 0x558e80233300 allocate 0x17000/1000,17000,0 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845c2c00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845c2c00 onodes shared_blobs 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x5b4000~17000/1000,17000,0 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845c2c00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x5b4000~17000] 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84549d50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d03490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84549d50 blob([0x5b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03490 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2c00 prepare 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845c2c00 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e84549d50) get_ref 0x0~2000 Blob(0x558e84549d50 blob([0x5b4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d03490 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2c00 io_done 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e84549d50) get_ref init 0x2000, 1000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3b000~2000: 0x0~2000 Blob(0x558e84549d50 blob([0x5b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d03490 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83d034a8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472f880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472f8f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472f880 blob([0x5b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472f8f0 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472f880) get_ref 0x0~2000 Blob(0x558e8472f880 blob([0x5b6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472f8f0 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472f880) get_ref init 0x2000, 1000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3d000~2000: 0x0~2000 Blob(0x558e8472f880 blob([0x5b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472f8f0 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472f908 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472f9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472fa40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472f9d0 blob([0x5b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472fa40 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472f9d0) get_ref 0x0~2000 Blob(0x558e8472f9d0 blob([0x5b8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472fa40 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472f9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3f000~2000: 0x0~2000 Blob(0x558e8472f9d0 blob([0x5b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472fa40 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472fa58 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472fab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472fb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472fab0 blob([0x5ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472fb20 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472fab0) get_ref 0x0~2000 Blob(0x558e8472fab0 blob([0x5ba000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472fb20 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472fab0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x41000~2000: 0x0~2000 Blob(0x558e8472fab0 blob([0x5ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472fb20 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472fb38 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472fb90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472fc00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472fb90 blob([0x5bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472fc00 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472fb90) get_ref 0x0~2000 Blob(0x558e8472fb90 blob([0x5bc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472fc00 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472fb90) get_ref init 0x2000, 1000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x43000~2000: 0x0~2000 Blob(0x558e8472fb90 blob([0x5bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472fc00 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472fc18 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472fc70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472fce0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472fc70 blob([0x5be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472fce0 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472fc70) get_ref 0x0~2000 Blob(0x558e8472fc70 blob([0x5be000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472fce0 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472fc70) get_ref init 0x2000, 1000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x45000~2000: 0x0~2000 Blob(0x558e8472fc70 blob([0x5be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472fce0 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472fcf8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472fd50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472fdc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472fd50 blob([0x5c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472fdc0 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472fd50) get_ref 0x0~2000 Blob(0x558e8472fd50 blob([0x5c0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472fdc0 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472fd50) get_ref init 0x2000, 1000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x47000~2000: 0x0~2000 Blob(0x558e8472fd50 blob([0x5c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472fdc0 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 5 cleaned 0 in 0.005342767s (0.000000080s flush + 0.005342687s kv commit) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472fdd8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 18 submitting 18 deferred done 0 stable 0 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472fe30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472fea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472fe30 blob([0x5c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472fea0 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472fe30) get_ref 0x0~2000 Blob(0x558e8472fe30 blob([0x5c2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472fea0 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472fe30) get_ref init 0x2000, 1000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x49000~2000: 0x0~2000 Blob(0x558e8472fe30 blob([0x5c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472fea0 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472feb8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472ff10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8472ff80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3471 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472ff10 blob([0x5c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472ff80 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472ff10) get_ref 0x0~2000 Blob(0x558e8472ff10 blob([0x5c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8472ff80 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e8472ff10) get_ref init 0x2000, 1000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4b000~2000: 0x0~2000 Blob(0x558e8472ff10 blob([0x5c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8472ff80 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8472ff98 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84754000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84754070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84754000 blob([0x5c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84754070 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e84754000) get_ref 0x0~2000 Blob(0x558e84754000 blob([0x5c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84754070 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e84754000) get_ref init 0x2000, 1000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4d000~2000: 0x0~2000 Blob(0x558e84754000 blob([0x5c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84754070 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84754088 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e847540e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84754150 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e847540e0 blob([0x5c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84754150 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e847540e0) get_ref 0x0~2000 Blob(0x558e847540e0 blob([0x5c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84754150 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e847540e0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4f000~2000: 0x0~2000 Blob(0x558e847540e0 blob([0x5c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84754150 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84754168 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03680 had 1 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e847541c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84754230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3680 had 1 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e847541c0 blob([!~1000,0x5ca000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84754230 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e847541c0) get_ref 0x1000~1000 Blob(0x558e847541c0 blob([!~1000,0x5ca000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84754230 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e846c3400 had 1 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.blob(0x558e847541c0) get_ref init 0x2000, 1000 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e846c3680 had 1 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x51000~1000: 0x1000~1000 Blob(0x558e847541c0 blob([!~1000,0x5ca000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558e84754230 sbid 0x0)) 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84754248 in 0x558e80219a40) _discard 0x1000~1000 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4664 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x52000 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 2 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811ddc88) dirty_range mark inline shard dirty 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5928 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x3b000~17000 = 0 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:06.677+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 7069 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 3 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.677+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000029.00000000000000000010' <- 0000000029.00000000000000000010 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.677+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8281 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:06.677+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 148186 (13 ios * 4000 + 96186 bytes) 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 2 2019-12-07T19:34:06.677+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes 0x558e811ddb80 shared_blobs 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811ddc88) update 3#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:34:06.677+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 9422 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 10445 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 3 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811ddc88) update inline shard 812 bytes from 42 extents 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 11539 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2c6a64700 10 osd.0 29 dequeue_op 0x558e818051e0 prio 196 cost 0 latency 0.000496 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9da56077:::smithi13140796-13:head# is 1303 (485 bytes onode + 2 bytes spanning blobs + 816 bytes inline extents) 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 2 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[5b4000~17000] released 0x[] 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 12649 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 freelist allocate 0x5b4000~17000 2019-12-07T19:34:06.677+0000 7fe2c7265700 20 freelist _xor first_key 0x580000 last_key 0x580000 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 13104 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 13683 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 2 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 16675 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:06.677+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811ddb80 had 2 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 2 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 17769 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2c6a64700 10 osd.0 29 dequeue_op 0x558e818051e0 finish 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 snap_mapper.get_snaps 1:891f0d13:::smithi13140796-14:4 4 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 snap_mapper.clear_snaps 1:891f0d13:::smithi13140796-14:4 2019-12-07T19:34:06.677+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 snap_mapper.clear_snaps rm OBJ_.1_0000000000000001.198F0B8C.4.smithi13140796-14.. 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 18910 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:34:06.677+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e819a1200 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 snap_mapper._remove_oid rm SNA_1_0000000000000004_.1_0000000000000001.198F0B8C.4.smithi13140796-14.. 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 2 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 27'7 (0'0,27'7] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=27'7 active mbc={} ps=[2~1]] append_log log((0'0,27'7], crt=27'7) [29'8 (27'4) delete 1:891f0d13:::smithi13140796-14:4 by unknown.0.0:0 2019-12-07T19:34:06.673165+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,29'9 (27'7) modify 1:891f0d13:::smithi13140796-14:head by unknown.0.0:0 2019-12-07T19:34:06.673165+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 20122 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 29'8 (0'0,29'8] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 lua=27'7 crt=27'7 lcod 27'7 active mbc={} ps=[2~1]] add_log_entry 29'8 (27'4) delete 1:891f0d13:::smithi13140796-14:4 by unknown.0.0:0 2019-12-07T19:34:06.673165+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 21263 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 2 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 lua=27'7 crt=27'7 lcod 27'7 active mbc={} ps=[2~1]] add_log_entry 29'9 (27'7) modify 1:891f0d13:::smithi13140796-14:head by unknown.0.0:0 2019-12-07T19:34:06.673165+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 22475 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 lua=27'7 crt=27'7 lcod 27'7 active mbc={} ps=[2~1]] append_log approx pg log length = 9 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 23569 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 lua=27'7 crt=27'7 lcod 27'7 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 2 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 24679 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:06.677+0000 7fe2cb26d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'8, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 24702 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 24702 < min_flush_size 524288 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xe4cd0~607e to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.677+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xe4cd0 0xe4cd0~607e to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~c0000 x_off 0xb4cd0 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xcd0 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.677+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd4e and padding block with 0x2b2 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e84559b00,0x558e845c2000,0x558e845c2300,0x558e845c2600,0x558e83c57800> 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559b00 kv_submitted 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e8417b200 seq 26 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84559b00 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559b00 finishing 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84559b00 onodes 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84559b00 done 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845c2300 kv_submitted 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84559b00 [] 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) get_onode oid 1#1:891f0d13:::smithi13140796-14:4#8 key 0x818000000000000001891f0d1321736d'ithi13140796-14!='0x00000000000000040000000000000008'o' 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2000 kv_submitted 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845c2000 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.1s1_head 0x558e819b01e0) r -2 v.len 0 2019-12-07T19:34:06.677+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# -> 1#1:891f0d13:::smithi13140796-14:4#8 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2000 finishing 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845c2000 onodes 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845c2000 done 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845c2900 kv_submitted 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4# -> 1#1:891f0d13:::smithi13140796-14:4#8 = 0 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845c2000 [] 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.677+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2 keys 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2300 kv_submitted 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845c2300 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2 keys = 0 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2300 finishing 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845c2300 onodes 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845c2300 done 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845c2600 kv_submitted 2019-12-07T19:34:06.677+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845c2300 [] 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2600 kv_submitted 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845c2600 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2600 finishing 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845c2600 onodes 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845c2600 done 2019-12-07T19:34:06.677+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845c2600 [] 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.198F0B8C.4.smithi13140796-14..' <- OBJ_.1_0000000000000001.198F0B8C.4.smithi13140796-14.. 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 kv_submitted 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c57800 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 finishing 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c57800 onodes 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57800 done 2019-12-07T19:34:06.677+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c57800 [] 2019-12-07T19:34:06.677+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.1_0000000000000001.198F0B8C.4.smithi13140796-14..' <- SNA_1_0000000000000004_.1_0000000000000001.198F0B8C.4.smithi13140796-14.. 2019-12-07T19:34:06.677+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.677+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:06.677+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.0000000029.00000000000000000008' <- 0000000029.00000000000000000008 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.0000000029.00000000000000000009' <- 0000000029.00000000000000000009 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 osd.0 29 queue_for_pg_delete on 1.bs3 e 29 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGDelete(1.bs3 e29) prio 5 cost 1048576 e29) 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417b200 cost 7531 (1 ios * 4000 + 3531 bytes) 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGDelete(1.bs3 e29) prio 5 cost 1048576 e29) queued 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417b200 onodes 0x558e811dd680,0x558e8417ca00 shared_blobs 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811dd788) update 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGDelete(1.bs3 e29) prio 5 cost 1048576 e29) pg 0x558e81928000 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 DeleteSome 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:891f0d13:::smithi13140796-14:head# is 55193 (54390 bytes onode + 2 bytes spanning blobs + 801 bytes inline extents) 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] do_delete_work 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e8417cb08) update 1#1:891f0d13:::smithi13140796-14:4#8 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:891f0d13:::smithi13140796-14:4#8 is 54515 (54317 bytes onode + 2 bytes spanning blobs + 196 bytes inline extents) 2019-12-07T19:34:06.677+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.677+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) collection_list 1.bs3_head start GHMIN end GHMAX max 30 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list range 0x837ffffffffffffffdd0000000 to 0x837ffffffffffffffde0000000 and 0x838000000000000001d0000000 to 0x838000000000000001e0000000 start GHMIN 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417b200 allocated 0x[] released 0x[] 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 prepare 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417b200 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 io_done 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 lua=27'7 crt=27'7 lcod 27'7 active mbc={} ps=[2~1]] op_applied version 29'9 2019-12-07T19:34:06.677+0000 7fe2cb26d700 10 osd.0 29 dequeue_op 0x558e84176580 finish 2019-12-07T19:34:06.677+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list pend 0x837ffffffffffffffde0000000 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list key 0x8380000000000000015000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list oid 3#1:d0000000::::head# end GHMAX 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list oid 3#1:d313ddd1:::smithi13140796-49:head# end GHMAX 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list oid 3#1:db85c185:::smithi13140796-18:head# end GHMAX 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list oid 3#1:dce63cc3:::smithi13140796-50:head# end GHMAX 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list key 0x838000000000000001e000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:06.677+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) collection_list 1.bs3_head start GHMAX end GHMAX max 30 = 0, ls.size() = 4, next = GHMAX 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] do_delete_work [3#1:d0000000::::head#,3#1:d313ddd1:::smithi13140796-49:head#,3#1:db85c185:::smithi13140796-18:head#,3#1:dce63cc3:::smithi13140796-50:head#] 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 snap_mapper.remove_oid 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 snap_mapper._remove_oid 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:06.677+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.677+0000 7fe2ca26b700 20 bluestore.onode(0x558e8125c280).flush flush cnt:1 2019-12-07T19:34:06.678+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 210 ==== MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=137, reqid=client.4540.0:246, at_version=29'11, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2294+0+0 (unknown 0 0 0) 0x558e819d8800 con 0x558e818d7c00 2019-12-07T19:34:06.678+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e83b0a000 prio 127 cost 0 latency 0.000032 epoch 29 MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=137, reqid=client.4540.0:246, at_version=29'11, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.678+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=137, reqid=client.4540.0:246, at_version=29'11, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=137, reqid=client.4540.0:246, at_version=29'11, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.678+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'10 (0'0,29'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'9 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] op_applied version 29'10 2019-12-07T19:34:06.678+0000 7fe2c7265700 10 osd.0 29 dequeue_op 0x558e84177600 finish 2019-12-07T19:34:06.678+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=137, reqid=client.4540.0:246, at_version=29'11, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558e8192b000 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 osd.0 29 dequeue_op 0x558e83b0a000 prio 127 cost 0 latency 0.000316 MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=137, reqid=client.4540.0:246, at_version=29'11, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s3( v 29'10 (0'0,29'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'10 (0'0,29'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=137, reqid=client.4540.0:246, at_version=29'11, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.678+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 aio_wait 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'10 (0'0,29'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] append_log log((0'0,29'10], crt=27'8) [29'11 (29'10) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:246 2019-12-07T19:34:05.667270+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.678+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:34:06.678+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'11 (0'0,29'11] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'10 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] add_log_entry 29'11 (29'10) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:246 2019-12-07T19:34:05.667270+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'11 (0'0,29'11] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'10 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] append_log approx pg log length = 11 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e29) pg 0x558e81a98000 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'11 (0'0,29'11] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'10 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.678+0000 7fe2cb26d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.942492559s) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'4 active pruub 44.060997103s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'4 active pruub 44.060997103s@ mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83d4d800 seq 28 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:06.678+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 3 keys 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 3 keys = 0 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e8417bb00 seq 22 2019-12-07T19:34:06.678+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.678+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:06.678+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000029.00000000000000000011' <- 0000000029.00000000000000000011 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417bb00 cost 5192 (1 ios * 4000 + 1192 bytes) 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417bb00 onodes shared_blobs 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._fastinfo' <- _fastinfo 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417bb00 prepare 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417bb00 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417bb00 io_done 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4d800 cost 5944 (1 ios * 4000 + 1944 bytes) 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4d800 onodes 0x558e811ddb80 shared_blobs 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e811ddc88) update 3#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9da56077:::smithi13140796-13:head# is 1303 (485 bytes onode + 2 bytes spanning blobs + 816 bytes inline extents) 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4d800 allocated 0x[] released 0x[] 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e29) pg 0x558e81a98000 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 prepare 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4d800 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 io_done 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'11 (0'0,29'11] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'10 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] op_applied version 29'11 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 pi=[18,28)/1 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.678+0000 7fe2cb26d700 10 osd.0 29 dequeue_op 0x558e83b0a000 finish 2019-12-07T19:34:06.678+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e8417af00 seq 23 2019-12-07T19:34:06.678+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417af00 cost 5257 (1 ios * 4000 + 1257 bytes) 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417af00 onodes shared_blobs 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417af00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 prepare 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417af00 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 io_done 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e29) pg 0x558e81a98000 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e8417a600 seq 24 2019-12-07T19:34:06.678+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417a600 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417a600 onodes shared_blobs 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417a600 allocated 0x[] released 0x[] 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 prepare 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417a600 2019-12-07T19:34:06.678+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 io_done 2019-12-07T19:34:06.678+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.678+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 116 ==== pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 ==== 53+0+0 (unknown 0 0 0) 0x558e81965000 con 0x558e818d7000 2019-12-07T19:34:06.678+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e83b0a2c0 prio 196 cost 0 latency 0.000015 epoch 29 pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 2019-12-07T19:34:06.678+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) 2019-12-07T19:34:06.678+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 117 ==== MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818ef000 con 0x558e818d7000 2019-12-07T19:34:06.678+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e83b0b4a0 prio 196 cost 0 latency 0.000012 epoch 29 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.678+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.678+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.678+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.678+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e83b0a2c0 prio 196 cost 0 latency 0.000141 pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.678+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 2019-12-07T19:34:06.678+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_update_log_missing_reply got reply from 3(5) 2019-12-07T19:34:06.678+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e83b0a2c0 finish 2019-12-07T19:34:06.678+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.678+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.678+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.678+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e83b0b4a0 prio 196 cost 0 latency 0.000111 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.678+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.678+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e83b0b4a0 finish 2019-12-07T19:34:06.678+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0xead4e 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 18 cleaned 0 in 0.003267959s (0.000004828s flush + 0.003263131s kv commit) 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 6 submitting 6 deferred done 0 stable 0 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 21170 < min_flush_size 524288 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 53938 < min_flush_size 524288 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 86706 < min_flush_size 524288 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 112188 < min_flush_size 524288 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dd680 had 1 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417ca00 had 1 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402f400 had 1 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 114631 < min_flush_size 524288 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811ddb80 had 2 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 2 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 115654 < min_flush_size 524288 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 3 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 118057 < min_flush_size 524288 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811ddb80 had 1 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 119151 < min_flush_size 524288 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 2 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 119474 < min_flush_size 524288 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 120272 < min_flush_size 524288 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 120295 < min_flush_size 524288 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 120295 < min_flush_size 524288 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0xead4e~1d5e7 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0xead4e 0xead4e~1d5e7 to file(ino 16 size 0xf0000 mtime 2019-12-07T19:34:06.664298+0000 allocated f0000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~c0000]) 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _allocate len 0x18335 from 0 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _allocate len 0x18335 from 1 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x20000/10000,20000,acd220000 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd220000~20000/10000,20000,acd220000 2019-12-07T19:34:06.680+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x110000 to include allocated 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 35 (was clean) 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xbad4e 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd4e 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x335 and padding block with 0xccb 2019-12-07T19:34:06.680+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x110000 2019-12-07T19:34:06.680+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 42 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83ebbc00 con 0x558e818d6000 2019-12-07T19:34:06.681+0000 7fe2e753b700 15 osd.0 29 enqueue_peering_evt 1.5s4 epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:06.681+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.681+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.681+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.681+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.681+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e29) pg 0x558e83d07000 2019-12-07T19:34:06.681+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:06.681+0000 7fe2c5261700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.681+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:06.681+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:06.681+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e819a0900 seq 15 2019-12-07T19:34:06.681+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:06.681+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 211 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1200+0+0 (unknown 0 0 0) 0x558e8122b100 con 0x558e818d7c00 2019-12-07T19:34:06.681+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._biginfo' <- _biginfo 2019-12-07T19:34:06.681+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:06.681+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:06.681+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0900 cost 5443 (1 ios * 4000 + 1443 bytes) 2019-12-07T19:34:06.681+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0900 onodes shared_blobs 2019-12-07T19:34:06.681+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.8s5 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:06.681+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0900 allocated 0x[] released 0x[] 2019-12-07T19:34:06.681+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.681+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 prepare 2019-12-07T19:34:06.681+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0900 2019-12-07T19:34:06.681+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 io_done 2019-12-07T19:34:06.681+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x108335 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.681+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.681+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.681+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.681+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e29) pg 0x558e841a4000 2019-12-07T19:34:06.681+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/18 les/c/f=29/19/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:06.681+0000 7fe2c7a66700 20 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] update_history advanced history from ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (35) on file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]), flushing log 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:06.681+0000 7fe2c7a66700 20 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 pi=[18,28)/1 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] update_history clearing past_intervals 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 35 len 0x49 crc 0x1406f1c9) 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x22000~1000 to file(ino 1 size 0x22000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x22000 0x22000~1000 to file(ino 1 size 0x22000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x23000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x22000 2019-12-07T19:34:06.681+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:06.681+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:06.681+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e8106c000 seq 26 2019-12-07T19:34:06.681+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:06.681+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._biginfo' <- _biginfo 2019-12-07T19:34:06.681+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:06.681+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:06.681+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c000 cost 5375 (1 ios * 4000 + 1375 bytes) 2019-12-07T19:34:06.681+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c000 onodes shared_blobs 2019-12-07T19:34:06.681+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c000 allocated 0x[] released 0x[] 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x23000 2019-12-07T19:34:06.681+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 prepare 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:06.681+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c000 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:34:06.681+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 io_done 2019-12-07T19:34:06.681+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000021 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.681+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 212 ==== pg_info2(1.as3 1.as3 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e8122b100 con 0x558e818d7c00 2019-12-07T19:34:06.681+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.as3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:06.681+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 35 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 6 cleaned 0 in 0.001334028s (0.000004587s flush + 0.001329441s kv commit) 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1280 < min_flush_size 524288 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2492 < min_flush_size 524288 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2515 < min_flush_size 524288 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2515 < min_flush_size 524288 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x108335~9d3 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.681+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x108335 0x108335~9d3 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd8335 2019-12-07T19:34:06.681+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x335 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.682+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd08 and padding block with 0x2f8 2019-12-07T19:34:06.682+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.682+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.682+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e29) pg 0x558e8193b000 2019-12-07T19:34:06.682+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:06.682+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.682+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:06.682+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:06.682+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e83e73800 seq 20 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x108d08 2019-12-07T19:34:06.682+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.682+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.682+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.682+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.682+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:06.682+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000115218s (0.000000084s flush + 0.000115134s kv commit) 2019-12-07T19:34:06.682+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.682+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e73800 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.682+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e73800 onodes shared_blobs 2019-12-07T19:34:06.682+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e73800 allocated 0x[] released 0x[] 2019-12-07T19:34:06.682+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73800 prepare 2019-12-07T19:34:06.682+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e73800 2019-12-07T19:34:06.682+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73800 io_done 2019-12-07T19:34:06.682+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.682+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:06.682+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.682+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.682+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.682+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x108d08~46d to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.682+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x108d08 0x108d08~46d to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd8d08 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd08 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x175 and padding block with 0xe8b 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x109175 2019-12-07T19:34:06.682+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.682+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.682+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000135348s (0.000000112s flush + 0.000135236s kv commit) 2019-12-07T19:34:06.682+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.682+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 213 ==== pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 ==== 53+0+0 (unknown 0 0 0) 0x558e819dee00 con 0x558e818d7c00 2019-12-07T19:34:06.682+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e81943e40 prio 196 cost 0 latency 0.000026 epoch 29 pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 2019-12-07T19:34:06.682+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) 2019-12-07T19:34:06.682+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 214 ==== MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e84028200 con 0x558e818d7c00 2019-12-07T19:34:06.682+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e81943a20 prio 196 cost 0 latency 0.000010 epoch 29 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.682+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.682+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.682+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.682+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.682+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.682+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e81943e40 prio 196 cost 0 latency 0.000198 pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.682+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: pg_update_log_missing_reply(1.2s0 epoch 29/28 rep_tid 25 lcod 29'13) v3 2019-12-07T19:34:06.682+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] do_update_log_missing_reply got reply from 1(2) 2019-12-07T19:34:06.682+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=27'12 crt=27'12 lcod 23'10 mlcod 0'0 active+clean+snaptrim trimq=[4~1] ps=[2~1]] repop_all_committed: repop tid 25 all committed 2019-12-07T19:34:06.682+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=29'13 crt=27'12 lcod 27'12 mlcod 27'12 active+clean+snaptrim trimq=[4~1] ps=[2~1]] eval_repop repgather(0x558e83dcc5a0 29'13 rep_tid=25 committed?=1 r=-2) 2019-12-07T19:34:06.682+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=29'13 crt=27'12 lcod 27'12 mlcod 27'12 active+clean+snaptrim trimq=[4~1] ps=[2~1]] commit: repgather(0x558e83dcc5a0 29'13 rep_tid=25 committed?=1 r=-2) 2019-12-07T19:34:06.682+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=29'13 crt=27'12 lcod 27'12 mlcod 27'12 active+clean+snaptrim trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.682+0000 7fe2caa6c700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=29'13 crt=27'12 lcod 27'12 mlcod 27'12 active+clean+snaptrim trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:84 2019-12-07T19:34:06.682+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=29'13 crt=27'12 lcod 27'12 mlcod 27'12 active+clean+snaptrim trimq=[4~1] ps=[2~1]] removing repgather(0x558e83dcc5a0 29'13 rep_tid=25 committed?=1 r=-2) 2019-12-07T19:34:06.682+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=29'13 crt=27'12 lcod 27'12 mlcod 27'12 active+clean+snaptrim trimq=[4~1] ps=[2~1]] q front is repgather(0x558e83dcc5a0 29'13 rep_tid=25 committed?=1 r=-2) 2019-12-07T19:34:06.683+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=29'13 crt=27'12 lcod 27'12 mlcod 27'12 active+clean+snaptrim trimq=[4~1] ps=[2~1]] finished operator() r=-2 2019-12-07T19:34:06.683+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=29'13 crt=27'12 lcod 27'12 mlcod 27'12 active+clean+snaptrim trimq=[4~1] ps=[2~1]] sending commit on osd_op(client.4540.0:249 1.2s0 1:41df217a:::smithi13140796-39:head [delete] snapc 6=[6] ondisk+write+peerstat_old+known_if_redirected e29) v8 0x558e8417c000 2019-12-07T19:34:06.683+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(249 smithi13140796-39 [delete] v29'13 uv12 ondisk = -2 ((2) No such file or directory)) v8 -- 0x558e8417c000 con 0x558e83d68000 2019-12-07T19:34:06.683+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=29'13 crt=27'12 lcod 27'12 mlcod 27'12 active+clean+snaptrim trimq=[4~1] ps=[2~1]] remove_repop repgather(0x558e83dcc5a0 29'13 rep_tid=25 committed?=1 r=-2) 2019-12-07T19:34:06.683+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e81943e40 finish 2019-12-07T19:34:06.683+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.683+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.683+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.683+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.683+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e81943a20 prio 196 cost 0 latency 0.000335 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=29'13 crt=27'12 lcod 27'12 mlcod 27'12 active+clean+snaptrim trimq=[4~1] ps=[2~1]] 2019-12-07T19:34:06.683+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=29'13 crt=27'12 lcod 27'12 mlcod 27'12 active+clean+snaptrim trimq=[4~1] ps=[2~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=26, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.683+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e81943a20 finish 2019-12-07T19:34:06.683+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.683+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 215 ==== MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=139, reqid=osd.1.0:139, at_version=29'13, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2632+0+0 (unknown 0 0 0) 0x558e83feb800 con 0x558e818d7c00 2019-12-07T19:34:06.683+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e8477a000 prio 127 cost 0 latency 0.000043 epoch 29 MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=139, reqid=osd.1.0:139, at_version=29'13, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.683+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=139, reqid=osd.1.0:139, at_version=29'13, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:06.684+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 snap_mapper.get_snaps 1:d313ddd1:::smithi13140796-49:head got.empty() 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 snap_mapper.remove_oid 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 snap_mapper._remove_oid 1:db85c185:::smithi13140796-18:head 2019-12-07T19:34:06.684+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:06.684+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 snap_mapper.get_snaps 1:db85c185:::smithi13140796-18:head got.empty() 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 snap_mapper.remove_oid 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 snap_mapper._remove_oid 1:dce63cc3:::smithi13140796-50:head 2019-12-07T19:34:06.684+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:06.684+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 snap_mapper.get_snaps 1:dce63cc3:::smithi13140796-50:head got.empty() 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] do_delete_work deleting 3 objects 2019-12-07T19:34:06.684+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.684+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e845c2f00 seq 31 2019-12-07T19:34:06.684+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# onode 0x558e8402ef00 txc 0x558e845c2f00 2019-12-07T19:34:06.684+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# 0x0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194e700) put_ref 0x0~2000 Blob(0x558e8194e700 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9ce0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194ecb0) put_ref 0x0~2000 Blob(0x558e8194ecb0 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194ee00 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194ed20) put_ref 0x0~2000 Blob(0x558e8194ed20 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194eee0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194ef50) put_ref 0x0~2000 Blob(0x558e8194ef50 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64ee0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d0fd50) put_ref 0x0~2000 Blob(0x558e83d0fd50 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fab0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194fb20) put_ref 0x0~2000 Blob(0x558e8194fb20 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f9d0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194e7e0) put_ref 0x0~2000 Blob(0x558e8194e7e0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d1880 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e811d09a0) put_ref 0x0~2000 Blob(0x558e811d09a0 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867490 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d02460) put_ref 0x0~2000 Blob(0x558e83d02460 blob([0x2b4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d024d0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83dd2000) put_ref 0x0~2000 Blob(0x558e83dd2000 blob([0x2b6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5c00 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac48c0) put_ref 0x0~2000 Blob(0x558e83ac48c0 blob([0x2b8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5c70 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e817e9110) put_ref 0x0~2000 Blob(0x558e817e9110 blob([0x2ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64c40 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83c64cb0) put_ref 0x0~2000 Blob(0x558e83c64cb0 blob([0x2bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c650a0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e84014d90) put_ref 0x0~1000 Blob(0x558e84014d90 blob([0x2be000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e84014e00 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e840152d0) put_ref 0x0~2000 Blob(0x558e840152d0 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015260 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015340) put_ref 0x0~2000 Blob(0x558e84015340 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840153b0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c770) put_ref 0x0~2000 Blob(0x558e83e6c770 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c7e0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c850) put_ref 0x0~2000 Blob(0x558e83e6c850 blob([0x2e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c8c0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c930) put_ref 0x0~2000 Blob(0x558e83e6c930 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c9a0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6ca10) put_ref 0x0~2000 Blob(0x558e83e6ca10 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6ca80 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6caf0) put_ref 0x0~2000 Blob(0x558e83e6caf0 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8af0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e817e9d50) put_ref 0x0~2000 Blob(0x558e817e9d50 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840151f0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015180) put_ref 0x0~2000 Blob(0x558e84015180 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cb60 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6cbd0) put_ref 0x0~2000 Blob(0x558e83e6cbd0 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cc40 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6ccb0) put_ref 0x0~2000 Blob(0x558e83e6ccb0 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6cd20 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6cd90) put_ref 0x1000~1000 Blob(0x558e83e6cd90 blob([!~1000,0x2f1000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,1000]) SharedBlob(0x558e83e6ce00 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e8402f008) dirty_range mark inline shard dirty 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e8194e700 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e817e9ce0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2a4000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194e700) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e817e9cf8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e83c8ebd0 space 0x558e817e9cf8 0x0~1000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d2630 space 0x558e817e9cf8 0x1000~1000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2a4000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e817e9ce0) put 0x558e817e9ce0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e817e9cf8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e8194ecb0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8194ee00 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2a6000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194ecb0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194ee18 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d2240 space 0x558e8194ee18 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2a6000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e8194ee00) put 0x558e8194ee00 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194ee18 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e8194ed20 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8194eee0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2a8000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194ed20) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194eef8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d22d0 space 0x558e8194eef8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2a8000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e8194eee0) put 0x558e8194eee0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194eef8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e8194ef50 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83c64ee0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2aa000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194ef50) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83c64ef8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d3200 space 0x558e83c64ef8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2aa000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83c64ee0) put 0x558e83c64ee0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83c64ef8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e83d0fd50 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8194fab0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2ac000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d0fd50) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194fac8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d34d0 space 0x558e8194fac8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2ac000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e8194fab0) put 0x558e8194fab0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194fac8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e8194fb20 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8194f9d0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2ae000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194fb20) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194f9e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d3290 space 0x558e8194f9e8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2ae000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e8194f9d0) put 0x558e8194f9d0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194f9e8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e8194e7e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e811d1880 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2b0000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194e7e0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e811d1898 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d26c0 space 0x558e811d1898 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2b0000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e811d1880) put 0x558e811d1880 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e811d1898 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558e811d09a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e81867490 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2b2000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e811d09a0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e818674a8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e84378900 space 0x558e818674a8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2b2000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e81867490) put 0x558e81867490 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e818674a8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558e83d02460 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d024d0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2b4000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d02460) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83d024e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d3440 space 0x558e83d024e8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2b4000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83d024d0) put 0x558e83d024d0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83d024e8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558e83dd2000 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83ac5c00 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2b6000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83dd2000) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5c18 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d3560 space 0x558e83ac5c18 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2b6000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83ac5c00) put 0x558e83ac5c00 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5c18 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558e83ac48c0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83ac5c70 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2b8000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac48c0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5c88 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d3320 space 0x558e83ac5c88 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2b8000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83ac5c70) put 0x558e83ac5c70 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5c88 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558e817e9110 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83c64c40 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2ba000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e817e9110) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83c64c58 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d2510 space 0x558e83c64c58 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2ba000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83c64c40) put 0x558e83c64c40 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83c64c58 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558e83c64cb0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83c650a0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2bc000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83c64cb0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83c650b8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d33b0 space 0x558e83c650b8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2bc000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83c650a0) put 0x558e83c650a0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83c650b8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1a000~1000: 0x0~1000 Blob(0x558e84014d90 blob([!~1000] csum crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558e84014e00 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2be000~1000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e84014d90) discard_unallocated 0x0~1000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84014e18 in 0x558e80219c00) _discard 0x0~1000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d3a70 space 0x558e84014e18 0x0~1000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2be000~1000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e84014e00) put 0x558e84014e00 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84014e18 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1b000~2000: 0x0~2000 Blob(0x558e840152d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84015260 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2db000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e840152d0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015278 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d3b90 space 0x558e84015278 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2db000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e84015260) put 0x558e84015260 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015278 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1d000~2000: 0x0~2000 Blob(0x558e84015340 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e840153b0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2dd000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015340) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e840153c8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d3c20 space 0x558e840153c8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2dd000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e840153b0) put 0x558e840153b0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e840153c8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1f000~2000: 0x0~2000 Blob(0x558e83e6c770 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6c7e0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2df000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c770) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c7f8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d35f0 space 0x558e83e6c7f8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2df000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6c7e0) put 0x558e83e6c7e0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c7f8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x21000~2000: 0x0~2000 Blob(0x558e83e6c850 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6c8c0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2e1000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c850) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c8d8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d3680 space 0x558e83e6c8d8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2e1000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6c8c0) put 0x558e83e6c8c0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c8d8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x23000~2000: 0x0~2000 Blob(0x558e83e6c930 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6c9a0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2e3000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c930) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c9b8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d3710 space 0x558e83e6c9b8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2e3000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6c9a0) put 0x558e83e6c9a0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c9b8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x25000~2000: 0x0~2000 Blob(0x558e83e6ca10 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6ca80 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2e5000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6ca10) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6ca98 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d37a0 space 0x558e83e6ca98 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2e5000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6ca80) put 0x558e83e6ca80 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6ca98 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x27000~2000: 0x0~2000 Blob(0x558e83e6caf0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e817e8af0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2e7000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6caf0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e817e8b08 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d3830 space 0x558e817e8b08 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2e7000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e817e8af0) put 0x558e817e8af0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e817e8b08 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x29000~2000: 0x0~2000 Blob(0x558e817e9d50 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e840151f0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2e9000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e817e9d50) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015208 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d25a0 space 0x558e84015208 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2e9000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e840151f0) put 0x558e840151f0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015208 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2b000~2000: 0x0~2000 Blob(0x558e84015180 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6cb60 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2eb000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015180) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6cb78 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d3b00 space 0x558e83e6cb78 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2eb000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6cb60) put 0x558e83e6cb60 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6cb78 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2d000~2000: 0x0~2000 Blob(0x558e83e6cbd0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6cc40 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2ed000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6cbd0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6cc58 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d38c0 space 0x558e83e6cc58 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2ed000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6cc40) put 0x558e83e6cc40 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6cc58 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2f000~2000: 0x0~2000 Blob(0x558e83e6ccb0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6cd20 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2ef000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6ccb0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6cd38 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d3950 space 0x558e83e6cd38 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2ef000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6cd20) put 0x558e83e6cd20 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6cd38 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x31000~1000: 0x1000~1000 Blob(0x558e83e6cd90 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6ce00 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2f1000~1000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6cd90) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6ce18 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843d39e0 space 0x558e83e6ce18 0x1000~1000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2f1000~1000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6ce00) put 0x558e83e6ce00 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6ce18 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.bs3_head 3#1:d313ddd1:::smithi13140796-49:head# = 0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# onode 0x558e83c73400 txc 0x558e845c2f00 2019-12-07T19:34:06.684+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# 0x0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac4cb0) put_ref 0x0~2000 Blob(0x558e83ac4cb0 blob([0x79000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4c40 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac4850) put_ref 0x0~2000 Blob(0x558e83ac4850 blob([0x7b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f650 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194f5e0) put_ref 0x0~2000 Blob(0x558e8194f5e0 blob([0x7d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4d90 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac4e00) put_ref 0x0~2000 Blob(0x558e83ac4e00 blob([0x7f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4e70 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac4ee0) put_ref 0x0~2000 Blob(0x558e83ac4ee0 blob([0x81000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4f50 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac4fc0) put_ref 0x0~2000 Blob(0x558e83ac4fc0 blob([0x83000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5030 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac50a0) put_ref 0x0~2000 Blob(0x558e83ac50a0 blob([0x85000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5110 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac5180) put_ref 0x0~2000 Blob(0x558e83ac5180 blob([0x87000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac51f0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac5260) put_ref 0x0~2000 Blob(0x558e83ac5260 blob([0x89000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac52d0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e83c73508) dirty_range mark inline shard dirty 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e83ac4cb0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83ac4c40 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x79000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac4cb0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4c58 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e81170bd0 space 0x558e83ac4c58 0x0~1000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e83d103f0 space 0x558e83ac4c58 0x1000~1000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x79000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83ac4c40) put 0x558e83ac4c40 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4c58 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e83ac4850 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8194f650 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x7b000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac4850) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194f668 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e83c8e630 space 0x558e8194f668 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x7b000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e8194f650) put 0x558e8194f650 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e8194f668 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e8194f5e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83ac4d90 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x7d000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e8194f5e0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4da8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e83c8ecf0 space 0x558e83ac4da8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x7d000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83ac4d90) put 0x558e83ac4d90 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4da8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e83ac4e00 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83ac4e70 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x7f000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac4e00) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4e88 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e83d11e60 space 0x558e83ac4e88 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x7f000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83ac4e70) put 0x558e83ac4e70 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4e88 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e83ac4ee0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83ac4f50 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x81000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac4ee0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4f68 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e83d11a70 space 0x558e83ac4f68 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x81000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83ac4f50) put 0x558e83ac4f50 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4f68 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e83ac4fc0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83ac5030 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x83000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac4fc0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5048 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e83d11dd0 space 0x558e83ac5048 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x83000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83ac5030) put 0x558e83ac5030 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5048 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e83ac50a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83ac5110 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x85000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac50a0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5128 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e83d11560 space 0x558e83ac5128 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x85000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83ac5110) put 0x558e83ac5110 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5128 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558e83ac5180 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83ac51f0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x87000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac5180) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5208 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843787e0 space 0x558e83ac5208 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x87000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83ac51f0) put 0x558e83ac51f0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac5208 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558e83ac5260 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83ac52d0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x89000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83ac5260) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac52e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e84378870 space 0x558e83ac52e8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x89000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83ac52d0) put 0x558e83ac52d0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac52e8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.bs3_head 3#1:db85c185:::smithi13140796-18:head# = 0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# onode 0x558e83cb2280 txc 0x558e845c2f00 2019-12-07T19:34:06.684+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# 0x0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d02620) put_ref 0x0~2000 Blob(0x558e83d02620 blob([0x2bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f48e00 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d37030) put_ref 0x0~2000 Blob(0x558e83d37030 blob([0x2c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4bd0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d0fb20) put_ref 0x0~2000 Blob(0x558e83d0fb20 blob([0x2c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fb90 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d0fc00) put_ref 0x0~2000 Blob(0x558e83d0fc00 blob([0x2c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fc70 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d0fce0) put_ref 0x0~2000 Blob(0x558e83d0fce0 blob([0x2c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015490 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015500) put_ref 0x0~2000 Blob(0x558e84015500 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015570 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e840155e0) put_ref 0x0~2000 Blob(0x558e840155e0 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015650 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e840156c0) put_ref 0x0~2000 Blob(0x558e840156c0 blob([0x2cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015730 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e840157a0) put_ref 0x0~2000 Blob(0x558e840157a0 blob([0x2cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015810 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015880) put_ref 0x0~2000 Blob(0x558e84015880 blob([0x2d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840158f0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015960) put_ref 0x0~2000 Blob(0x558e84015960 blob([0x2d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840159d0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015a40) put_ref 0x0~2000 Blob(0x558e84015a40 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015ab0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015b20) put_ref 0x0~2000 Blob(0x558e84015b20 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015b90 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015c00) put_ref 0x0~2000 Blob(0x558e84015c00 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84015c70 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e84030a10) put_ref 0x0~2000 Blob(0x558e84030a10 blob([0x2f2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fa40 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c310) put_ref 0x0~2000 Blob(0x558e83e6c310 blob([0x2f4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c380 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c460) put_ref 0x0~2000 Blob(0x558e83e6c460 blob([0x2f6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c4d0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c540) put_ref 0x0~2000 Blob(0x558e83e6c540 blob([0x2f8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c5b0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c620) put_ref 0x0~2000 Blob(0x558e83e6c620 blob([0x2fa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6c690 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c700) put_ref 0x0~2000 Blob(0x558e83e6c700 blob([0x2fc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6df80 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6df10) put_ref 0x0~2000 Blob(0x558e83e6df10 blob([0x2fe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dea0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6de30) put_ref 0x0~2000 Blob(0x558e83e6de30 blob([0x300000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6ddc0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6dd50) put_ref 0x0~2000 Blob(0x558e83e6dd50 blob([0x302000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dce0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6dc70) put_ref 0x0~2000 Blob(0x558e83e6dc70 blob([0x304000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6dc00 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6db90) put_ref 0x0~2000 Blob(0x558e83e6db90 blob([0x306000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6db20 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6dab0) put_ref 0x0~1000 Blob(0x558e83e6dab0 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83e6da40 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.extentmap(0x558e83cb2388) dirty_range mark inline shard dirty 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e83d02620 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f48e00 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2bf000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d02620) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83f48e18 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e84032360 space 0x558e83f48e18 0x0~1000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843ba6c0 space 0x558e83f48e18 0x1000~1000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2bf000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83f48e00) put 0x558e83f48e00 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83f48e18 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e83d37030 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83ac4bd0 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2c1000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d37030) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4be8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843ba900 space 0x558e83ac4be8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2c1000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83ac4bd0) put 0x558e83ac4bd0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83ac4be8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e83d0fb20 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d0fb90 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2c3000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d0fb20) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83d0fba8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843ba750 space 0x558e83d0fba8 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2c3000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83d0fb90) put 0x558e83d0fb90 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83d0fba8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e83d0fc00 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d0fc70 sbid 0x0)) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2c5000~2000] 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d0fc00) discard_unallocated 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83d0fc88 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 _rm buffer(0x558e843ba7e0 space 0x558e83d0fc88 0x0~2000 clean) 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2c5000~2000 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83d0fc70) put 0x558e83d0fc70 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83d0fc88 in 0x558e80219c00) _clear 2019-12-07T19:34:06.684+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e83d0fce0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84015490 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2c7000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e83d0fce0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e840154a8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843ba870 space 0x558e840154a8 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2c7000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e84015490) put 0x558e84015490 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e840154a8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e84015500 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84015570 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2c9000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015500) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015588 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bafc0 space 0x558e84015588 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2c9000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e84015570) put 0x558e84015570 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015588 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e840155e0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84015650 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2cb000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e840155e0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015668 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bb050 space 0x558e84015668 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2cb000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e84015650) put 0x558e84015650 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015668 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558e840156c0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84015730 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2cd000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e840156c0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015748 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bb0e0 space 0x558e84015748 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2cd000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e84015730) put 0x558e84015730 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015748 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558e840157a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84015810 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2cf000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e840157a0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015828 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bb170 space 0x558e84015828 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2cf000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e84015810) put 0x558e84015810 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015828 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558e84015880 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e840158f0 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2d1000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015880) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015908 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bb200 space 0x558e84015908 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2d1000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e840158f0) put 0x558e840158f0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015908 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558e84015960 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e840159d0 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2d3000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015960) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e840159e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bb290 space 0x558e840159e8 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2d3000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e840159d0) put 0x558e840159d0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e840159e8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558e84015a40 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84015ab0 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2d5000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015a40) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015ac8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bb320 space 0x558e84015ac8 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2d5000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e84015ab0) put 0x558e84015ab0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015ac8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558e84015b20 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84015b90 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2d7000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015b20) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015ba8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bb3b0 space 0x558e84015ba8 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2d7000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e84015b90) put 0x558e84015b90 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015ba8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558e84015c00 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84015c70 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2d9000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e84015c00) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015c88 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bb440 space 0x558e84015c88 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2d9000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e84015c70) put 0x558e84015c70 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e84015c88 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558e84030a10 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d0fa40 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2f2000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e84030a10) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83d0fa58 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bb4d0 space 0x558e83d0fa58 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2f2000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83d0fa40) put 0x558e83d0fa40 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83d0fa58 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558e83e6c310 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6c380 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2f4000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c310) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c398 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843ba990 space 0x558e83e6c398 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2f4000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6c380) put 0x558e83e6c380 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c398 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558e83e6c460 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6c4d0 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2f6000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c460) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c4e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843baa20 space 0x558e83e6c4e8 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2f6000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6c4d0) put 0x558e83e6c4d0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c4e8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558e83e6c540 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6c5b0 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2f8000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c540) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c5c8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843baab0 space 0x558e83e6c5c8 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2f8000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6c5b0) put 0x558e83e6c5b0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c5c8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558e83e6c620 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6c690 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2fa000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c620) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c6a8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bab40 space 0x558e83e6c6a8 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2fa000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6c690) put 0x558e83e6c690 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6c6a8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x26000~2000: 0x0~2000 Blob(0x558e83e6c700 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6df80 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2fc000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6c700) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6df98 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843babd0 space 0x558e83e6df98 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2fc000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6df80) put 0x558e83e6df80 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6df98 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x28000~2000: 0x0~2000 Blob(0x558e83e6df10 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6dea0 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x2fe000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6df10) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6deb8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843baf30 space 0x558e83e6deb8 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x2fe000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6dea0) put 0x558e83e6dea0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6deb8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2a000~2000: 0x0~2000 Blob(0x558e83e6de30 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6ddc0 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x300000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6de30) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6ddd8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843baea0 space 0x558e83e6ddd8 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x300000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6ddc0) put 0x558e83e6ddc0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6ddd8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2c000~2000: 0x0~2000 Blob(0x558e83e6dd50 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6dce0 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x302000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6dd50) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6dcf8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bae10 space 0x558e83e6dcf8 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x302000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6dce0) put 0x558e83e6dce0 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6dcf8 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2e000~2000: 0x0~2000 Blob(0x558e83e6dc70 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6dc00 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x304000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6dc70) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6dc18 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bad80 space 0x558e83e6dc18 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x304000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6dc00) put 0x558e83e6dc00 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6dc18 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x30000~2000: 0x0~2000 Blob(0x558e83e6db90 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6db20 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x306000~2000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6db90) discard_unallocated 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6db38 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bacf0 space 0x558e83e6db38 0x0~2000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x306000~2000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6db20) put 0x558e83e6db20 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6db38 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x32000~1000: 0x0~1000 Blob(0x558e83e6dab0 blob([!~1000] csum crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558e83e6da40 sbid 0x0)) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x308000~1000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.blob(0x558e83e6dab0) discard_unallocated 0x0~1000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6da58 in 0x558e80219c00) _discard 0x0~1000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 _rm buffer(0x558e843bac60 space 0x558e83e6da58 0x0~1000 clean) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x308000~1000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.sharedblob(0x558e83e6da40) put 0x558e83e6da40 removing self from set 0x558e819c42b0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore.BufferSpace(0x558e83e6da58 in 0x558e80219c00) _clear 2019-12-07T19:34:06.685+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.bs3_head 3#1:dce63cc3:::smithi13140796-50:head# = 0 2019-12-07T19:34:06.685+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845c2f00 cost 4496 (1 ios * 4000 + 496 bytes) 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845c2f00 onodes shared_blobs 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845c2f00 allocated 0x[] released 0x[79000~12000,2a4000~65000] 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv release 0x79000~12000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 10 freelist release 0x79000~12000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 freelist _xor first_key 0x0 last_key 0x80000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv release 0x2a4000~65000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 10 freelist release 0x2a4000~65000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 freelist _xor first_key 0x280000 last_key 0x300000 2019-12-07T19:34:06.685+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2f00 prepare 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845c2f00 2019-12-07T19:34:06.685+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2f00 io_done 2019-12-07T19:34:06.685+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.685+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.685+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.685+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 354 < min_flush_size 524288 2019-12-07T19:34:06.685+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c73400 had 1 2019-12-07T19:34:06.685+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2280 had 1 2019-12-07T19:34:06.685+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ef00 had 1 2019-12-07T19:34:06.685+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 377 < min_flush_size 524288 2019-12-07T19:34:06.685+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 377 < min_flush_size 524288 2019-12-07T19:34:06.685+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.685+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x109175~179 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.685+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x109175 0x109175~179 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.685+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.685+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd9175 2019-12-07T19:34:06.685+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x175 2019-12-07T19:34:06.685+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.685+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x2ee and padding block with 0xd12 2019-12-07T19:34:06.685+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1092ee 2019-12-07T19:34:06.685+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.685+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.685+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.685+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.685+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000429477s (0.000000148s flush + 0.000429329s kv commit) 2019-12-07T19:34:06.685+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.685+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.685+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=139, reqid=osd.1.0:139, at_version=29'13, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.685+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.685+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=139, reqid=osd.1.0:139, at_version=29'13, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558e8192b000 2019-12-07T19:34:06.685+0000 7fe2c7265700 10 osd.0 29 dequeue_op 0x558e8477a000 prio 127 cost 0 latency 0.002249 MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=139, reqid=osd.1.0:139, at_version=29'13, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.9s3( v 29'11 (0'0,29'11] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.685+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'11 (0'0,29'11] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=139, reqid=osd.1.0:139, at_version=29'13, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.685+0000 7fe2c7265700 20 snap_mapper.remove_oid 1:9694a0e5:::smithi13140796-3:4 2019-12-07T19:34:06.685+0000 7fe2c7265700 20 snap_mapper._remove_oid 1:9694a0e5:::smithi13140796-3:4 2019-12-07T19:34:06.685+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.685+0000 7fe2c7265700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:06.685+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.685+0000 7fe2c7265700 20 snap_mapper.get_snaps 1:9694a0e5:::smithi13140796-3:4 4 2019-12-07T19:34:06.685+0000 7fe2c7265700 20 snap_mapper.clear_snaps 1:9694a0e5:::smithi13140796-3:4 2019-12-07T19:34:06.685+0000 7fe2c7265700 20 snap_mapper.clear_snaps rm OBJ_.3_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:06.685+0000 7fe2c7265700 20 snap_mapper._remove_oid rm SNA_1_0000000000000004_.3_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:06.685+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'11 (0'0,29'11] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] append_log log((0'0,29'11], crt=27'8) [29'12 (27'7) delete 1:9694a0e5:::smithi13140796-3:4 by unknown.0.0:0 2019-12-07T19:34:06.683044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,29'13 (27'8) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:06.683044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.685+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'11 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] add_log_entry 29'12 (27'7) delete 1:9694a0e5:::smithi13140796-3:4 by unknown.0.0:0 2019-12-07T19:34:06.683044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.685+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'11 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] add_log_entry 29'13 (27'8) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:06.683044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.685+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'11 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] append_log approx pg log length = 13 2019-12-07T19:34:06.685+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'11 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.685+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.685+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'12, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.686+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83c57500 seq 29 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) get_onode oid 3#1:9694a0e5:::smithi13140796-3:4#c key 0x8380000000000000019694a0e521736d'ithi13140796-3!='0x0000000000000004000000000000000c'o' 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) r -2 v.len 0 2019-12-07T19:34:06.686+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# -> 3#1:9694a0e5:::smithi13140796-3:4#c 2019-12-07T19:34:06.686+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4# -> 3#1:9694a0e5:::smithi13140796-3:4#c = 0 2019-12-07T19:34:06.686+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 2 keys 2019-12-07T19:34:06.686+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# 2 keys = 0 2019-12-07T19:34:06.686+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.686+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.686+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.3_0000000000000001.9692507A.4.smithi13140796-3..' <- OBJ_.3_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:06.686+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.686+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.3_0000000000000001.9692507A.4.smithi13140796-3..' <- SNA_1_0000000000000004_.3_0000000000000001.9692507A.4.smithi13140796-3.. 2019-12-07T19:34:06.686+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.686+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000029.00000000000000000012' <- 0000000029.00000000000000000012 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000029.00000000000000000013' <- 0000000029.00000000000000000013 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.686+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:06.686+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c57500 cost 7506 (1 ios * 4000 + 3506 bytes) 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c57500 onodes 0x558e83b08280,0x558e8402f180 shared_blobs 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore.extentmap(0x558e83b08388) update 3#1:9694a0e5:::smithi13140796-3:head# 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9694a0e5:::smithi13140796-3:head# is 835 (484 bytes onode + 2 bytes spanning blobs + 349 bytes inline extents) 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore.extentmap(0x558e8402f288) update 3#1:9694a0e5:::smithi13140796-3:4#c 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9694a0e5:::smithi13140796-3:4#c is 469 (409 bytes onode + 2 bytes spanning blobs + 58 bytes inline extents) 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c57500 allocated 0x[] released 0x[] 2019-12-07T19:34:06.686+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 prepare 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c57500 2019-12-07T19:34:06.686+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 io_done 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.686+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'11 crt=27'8 lcod 27'8 active mbc={} ps=[2~1]] op_applied version 29'13 2019-12-07T19:34:06.686+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3725 < min_flush_size 524288 2019-12-07T19:34:06.686+0000 7fe2c7265700 10 osd.0 29 dequeue_op 0x558e8477a000 finish 2019-12-07T19:34:06.686+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.686+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b08280 had 1 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402f180 had 1 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b08500 had 1 2019-12-07T19:34:06.686+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3748 < min_flush_size 524288 2019-12-07T19:34:06.686+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3748 < min_flush_size 524288 2019-12-07T19:34:06.686+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.686+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1092ee~ea4 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.686+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1092ee 0x1092ee~ea4 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xd92ee 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x2ee 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x192 and padding block with 0xe6e 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x10a192 2019-12-07T19:34:06.686+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.686+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.686+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000151006s (0.000000121s flush + 0.000150885s kv commit) 2019-12-07T19:34:06.686+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.686+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 216 ==== MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=138, reqid=osd.1.0:138, at_version=29'25, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2656+0+0 (unknown 0 0 0) 0x558e83feb000 con 0x558e818d7c00 2019-12-07T19:34:06.686+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e8477a160 prio 127 cost 0 latency 0.000034 epoch 29 MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=138, reqid=osd.1.0:138, at_version=29'25, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.686+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=138, reqid=osd.1.0:138, at_version=29'25, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.687+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 217 ==== MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=140, reqid=osd.1.0:140, at_version=29'27, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 1799+0+0 (unknown 0 0 0) 0x558e83fea800 con 0x558e818d7c00 2019-12-07T19:34:06.687+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e8477a2c0 prio 127 cost 0 latency 0.000029 epoch 29 MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=140, reqid=osd.1.0:140, at_version=29'27, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.687+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=140, reqid=osd.1.0:140, at_version=29'27, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=138, reqid=osd.1.0:138, at_version=29'25, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=138, reqid=osd.1.0:138, at_version=29'25, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558e841a4000 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 osd.0 29 dequeue_op 0x558e8477a160 prio 127 cost 0 latency 0.000132 MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=138, reqid=osd.1.0:138, at_version=29'25, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=28/29 n=8 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=138, reqid=osd.1.0:138, at_version=29'25, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 snap_mapper.remove_oid 1:10779340:::smithi13140796-32:4 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 snap_mapper._remove_oid 1:10779340:::smithi13140796-32:4 2019-12-07T19:34:06.687+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 snap_mapper.get_snaps 1:10779340:::smithi13140796-32:4 4 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 snap_mapper.clear_snaps 1:10779340:::smithi13140796-32:4 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 snap_mapper.clear_snaps rm OBJ_.5_0000000000000001.80EE9C20.4.smithi13140796-32.. 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 snap_mapper._remove_oid rm SNA_1_0000000000000004_.5_0000000000000001.80EE9C20.4.smithi13140796-32.. 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 27'23 (0'0,27'23] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] append_log log((0'0,27'23], crt=27'23) [29'24 (27'15) delete 1:10779340:::smithi13140796-32:4 by unknown.0.0:0 2019-12-07T19:34:06.682769+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,29'25 (27'18) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:06.682769+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'24 (0'0,29'24] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=27'23 crt=27'23 lcod 27'23 active mbc={} ps=[2~1]] add_log_entry 29'24 (27'15) delete 1:10779340:::smithi13140796-32:4 by unknown.0.0:0 2019-12-07T19:34:06.682769+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'25 (0'0,29'25] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=27'23 crt=27'23 lcod 27'23 active mbc={} ps=[2~1]] add_log_entry 29'25 (27'18) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:06.682769+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'25 (0'0,29'25] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=27'23 crt=27'23 lcod 27'23 active mbc={} ps=[2~1]] append_log approx pg log length = 25 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'25 (0'0,29'25] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=27'23 crt=27'23 lcod 27'23 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.687+0000 7fe2cba6e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'24, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e8195fb00 seq 27 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:10779340:::smithi13140796-32:4#18 key 0x858000000000000001107793'@!smithi13140796-32!='0x00000000000000040000000000000018'o' 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:06.687+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.8s5_head 5#1:10779340:::smithi13140796-32:4# -> 5#1:10779340:::smithi13140796-32:4#18 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.8s5_head 5#1:10779340:::smithi13140796-32:4# -> 5#1:10779340:::smithi13140796-32:4#18 = 0 2019-12-07T19:34:06.687+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:10779340:::smithi13140796-32:head# 2 keys = 0 2019-12-07T19:34:06.687+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.8s5_head 5#1:10779340:::smithi13140796-32:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.687+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.80EE9C20.4.smithi13140796-32..' <- OBJ_.5_0000000000000001.80EE9C20.4.smithi13140796-32.. 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.687+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.5_0000000000000001.80EE9C20.4.smithi13140796-32..' <- SNA_1_0000000000000004_.5_0000000000000001.80EE9C20.4.smithi13140796-32.. 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.687+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000029.00000000000000000024' <- 0000000029.00000000000000000024 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000029.00000000000000000025' <- 0000000029.00000000000000000025 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195fb00 cost 7531 (1 ios * 4000 + 3531 bytes) 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195fb00 onodes 0x558e8417d400,0x558e84315680 shared_blobs 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e8417d508) update 5#1:10779340:::smithi13140796-32:4#18 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:10779340:::smithi13140796-32:4#18 is 725 (412 bytes onode + 2 bytes spanning blobs + 311 bytes inline extents) 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e84315788) update 5#1:10779340:::smithi13140796-32:head# 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:10779340:::smithi13140796-32:head# is 1384 (485 bytes onode + 2 bytes spanning blobs + 897 bytes inline extents) 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195fb00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 prepare 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195fb00 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 io_done 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'25 (0'0,29'25] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=27'23 crt=27'23 lcod 27'23 active mbc={} ps=[2~1]] op_applied version 29'25 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 osd.0 29 dequeue_op 0x558e8477a160 finish 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=140, reqid=osd.1.0:140, at_version=29'27, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=140, reqid=osd.1.0:140, at_version=29'27, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558e841a4000 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 osd.0 29 dequeue_op 0x558e8477a2c0 prio 127 cost 0 latency 0.000393 MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=140, reqid=osd.1.0:140, at_version=29'27, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.8s5( v 29'25 (0'0,29'25] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=27'23 lcod 27'23 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.687+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'25 (0'0,29'25] local-lis/les=28/29 n=7 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=27'23 lcod 27'23 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=140, reqid=osd.1.0:140, at_version=29'27, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 snap_mapper.remove_oid 1:15219530:::smithi13140796-21:4 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 snap_mapper._remove_oid 1:15219530:::smithi13140796-21:4 2019-12-07T19:34:06.687+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.687+0000 7fe2cba6e700 20 bluestore.onode(0x558e8125c280).flush flush cnt:1 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e73500,0x558e845c3b00,0x558e845d4000,0x558e83d4c600,0x558e83d4c000,0x558e81a0a300,0x558e8195f500,0x558e81a0b500,0x558e81a0a900,0x558e84558f00,0x558e819a0000,0x558e84559200,0x558e83d4cf00,0x558e8417b800,0x558e81a0b800,0x558e811fa300,0x558e845c2900,0x558e845c2c00,0x558e8417b200,0x558e819a1200,0x558e8417bb00,0x558e83d4d800,0x558e8417af00,0x558e8417a600,0x558e819a0900,0x558e8106c000,0x558e83e73800,0x558e845c2f00,0x558e83c57500> 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73500 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e73500 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73500 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e73500 onodes 0x558e81a03680,0x558e83cb3680 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e73500 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e73500 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c3b00 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845c3b00 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c3b00 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845c3b00 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845c3b00 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845d4000 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845c3b00 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4000 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845d4000 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4000 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845d4000 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845d4000 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0900 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845d4000 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c600 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c600 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c600 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c600 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c600 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c000 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c600 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c000 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c000 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c000 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c000 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0000 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c000 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a300 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a300 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a300 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c000 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a300 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195f500 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195f500 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f500 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b500 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195f500 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b500 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b500 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b500 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a900 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b500 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a900 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a900 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a900 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a900 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558f00 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84558f00 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558f00 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84558f00 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84558f00 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84559200 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84558f00 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0000 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0000 onodes 0x558e811ddb80 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b282b8 in 0x558e80219a40) _finish_write discard buffer(0x558e846da6c0 space 0x558e83b282b8 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84030c58 in 0x558e80219a40) _finish_write discard buffer(0x558e83c8eea0 space 0x558e84030c58 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841db518 in 0x558e80219a40) _finish_write discard buffer(0x558e84251950 space 0x558e841db518 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841db5f8 in 0x558e80219a40) _finish_write discard buffer(0x558e842519e0 space 0x558e841db5f8 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841db6d8 in 0x558e80219a40) _finish_write discard buffer(0x558e846db5f0 space 0x558e841db6d8 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423c5c8 in 0x558e80219a40) _finish_write discard buffer(0x558e842518c0 space 0x558e8423c5c8 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472e018 in 0x558e80219a40) _finish_write discard buffer(0x558e846db560 space 0x558e8472e018 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472e0f8 in 0x558e80219a40) _finish_write discard buffer(0x558e846db4d0 space 0x558e8472e0f8 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472e1d8 in 0x558e80219a40) _finish_write discard buffer(0x558e846db440 space 0x558e8472e1d8 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472e2b8 in 0x558e80219a40) _finish_write discard buffer(0x558e846db3b0 space 0x558e8472e2b8 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472e398 in 0x558e80219a40) _finish_write discard buffer(0x558e846db320 space 0x558e8472e398 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472e478 in 0x558e80219a40) _finish_write discard buffer(0x558e846db290 space 0x558e8472e478 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472e558 in 0x558e80219a40) _finish_write discard buffer(0x558e846db200 space 0x558e8472e558 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472e638 in 0x558e80219a40) _finish_write discard buffer(0x558e846db170 space 0x558e8472e638 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472e718 in 0x558e80219a40) _finish_write discard buffer(0x558e846db0e0 space 0x558e8472e718 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472e7f8 in 0x558e80219a40) _finish_write discard buffer(0x558e846db050 space 0x558e8472e7f8 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472e8d8 in 0x558e80219a40) _finish_write discard buffer(0x558e846db680 space 0x558e8472e8d8 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472e9b8 in 0x558e80219a40) _finish_write discard buffer(0x558e846db710 space 0x558e8472e9b8 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472ea98 in 0x558e80219a40) _finish_write discard buffer(0x558e846db7a0 space 0x558e8472ea98 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472eb78 in 0x558e80219a40) _finish_write discard buffer(0x558e846db830 space 0x558e8472eb78 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472ec58 in 0x558e80219a40) _finish_write discard buffer(0x558e846db8c0 space 0x558e8472ec58 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472ed38 in 0x558e80219a40) _finish_write discard buffer(0x558e83e75b00 space 0x558e8472ed38 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472ee18 in 0x558e80219a40) _finish_write discard buffer(0x558e83e75b90 space 0x558e8472ee18 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0000 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0000 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559200 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84559200 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559200 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84559200 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84559200 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e73800 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84559200 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4cf00 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cf00 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4cf00 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4cf00 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b800 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4cf00 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417b800 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417b800 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b800 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b200 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417b800 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b800 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b800 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa300 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b800 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa300 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa300 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa300 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa300 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2900 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845c2900 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2900 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845c2900 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845c2900 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845c2c00 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845c2900 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2c00 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845c2c00 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2c00 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845c2c00 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845c2c00 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845c2c00 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417b200 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417b200 onodes 0x558e811dd680,0x558e8417ca00 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b200 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417b200 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 0x558e811ddb80 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d034a8 in 0x558e80219a40) _finish_write discard buffer(0x558e846da750 space 0x558e83d034a8 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472f908 in 0x558e80219a40) _finish_write discard buffer(0x558e846db950 space 0x558e8472f908 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472fa58 in 0x558e80219a40) _finish_write discard buffer(0x558e846db9e0 space 0x558e8472fa58 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472fb38 in 0x558e80219a40) _finish_write discard buffer(0x558e846dba70 space 0x558e8472fb38 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472fc18 in 0x558e80219a40) _finish_write discard buffer(0x558e846dbb00 space 0x558e8472fc18 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472fcf8 in 0x558e80219a40) _finish_write discard buffer(0x558e846dbb90 space 0x558e8472fcf8 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472fdd8 in 0x558e80219a40) _finish_write discard buffer(0x558e846dbc20 space 0x558e8472fdd8 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472feb8 in 0x558e80219a40) _finish_write discard buffer(0x558e846dbcb0 space 0x558e8472feb8 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8472ff98 in 0x558e80219a40) _finish_write discard buffer(0x558e846dbd40 space 0x558e8472ff98 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84754088 in 0x558e80219a40) _finish_write discard buffer(0x558e846dbdd0 space 0x558e84754088 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84754168 in 0x558e80219a40) _finish_write discard buffer(0x558e846dbe60 space 0x558e84754168 0x0~2000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84754248 in 0x558e80219a40) _finish_write discard buffer(0x558e846dbef0 space 0x558e84754248 0x1000~1000 writing nocache) 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4d800 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417bb00 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417bb00 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417bb00 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417bb00 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417bb00 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417af00 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417bb00 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4d800 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4d800 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4d800 onodes 0x558e811ddb80 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4d800 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57500 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4d800 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417af00 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417af00 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417af00 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a600 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417af00 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417a600 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417a600 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a600 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417a600 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0900 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0900 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0900 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0900 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c000 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c000 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c000 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c000 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 kv_queued 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c000 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73800 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e73800 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73800 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e73800 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e73800 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e73800 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2f00 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845c2f00 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c2f00 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845c2f00 onodes 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845c2f00 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845c2f00 [79000~12000,2a4000~65000] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release 0x79000~12000 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release 0x2a4000~65000 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c57500 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c57500 onodes 0x558e83b08280,0x558e8402f180 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57500 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c57500 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.688+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:06.688+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.688+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.688+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.688+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 29/28 ECSubWriteReply(tid=135, last_complete=29'9, committed=1, applied=1)) v2 -- 0x558e81238a00 con 0x558e818d7c00 2019-12-07T19:34:06.688+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 29/28 ECSubWriteReply(tid=86, last_complete=29'9, committed=1, applied=1)) v2 -- 0x558e84053a00 con 0x558e818d7000 2019-12-07T19:34:06.688+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 29/28 ECSubWriteReply(tid=136, last_complete=29'10, committed=1, applied=1)) v2 -- 0x558e84053400 con 0x558e818d7c00 2019-12-07T19:34:06.688+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 29/28 ECSubWriteReply(tid=137, last_complete=29'11, committed=1, applied=1)) v2 -- 0x558e83fefe00 con 0x558e818d7c00 2019-12-07T19:34:06.688+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 29/28 ECSubWriteReply(tid=139, last_complete=29'13, committed=1, applied=1)) v2 -- 0x558e81238c00 con 0x558e818d7c00 2019-12-07T19:34:06.688+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.688+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=29'13 crt=27'12 lcod 27'12 mlcod 27'12 active+clean+snaptrim trimq=[4~1] ps=[2~1]] handle_sub_write_reply Calling on_all_commit on Op(1:41df217a:::smithi13140796-39:4 v=29'15 tt=0'0 tid=26 reqid=osd.0.0:26 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:4=[],1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 luod=29'13 crt=27'12 lcod 27'12 mlcod 27'12 active+clean+snaptrim trimq=[4~1] ps=[2~1]] repop_all_committed: repop tid 26 all committed 2019-12-07T19:34:06.688+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] eval_repop repgather(0x558e83dccb40 29'15 rep_tid=26 committed?=1 r=0) (no op) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] commit: repgather(0x558e83dccb40 29'15 rep_tid=26 committed?=1 r=0) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.688+0000 7fe2caa6c700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:85 2019-12-07T19:34:06.688+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] removing repgather(0x558e83dccb40 29'15 rep_tid=26 committed?=1 r=0) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] q front is repgather(0x558e83dccb40 29'15 rep_tid=26 committed?=1 r=0) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] exit Trimming/WaitRepops 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] enter Trimming/WaitTrimTimer 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] exit Trimming/WaitTrimTimer 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:06.688+0000 7fe2caa6c700 10 osd.0 29 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] for snaptrim 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=29) prio 5 cost 1048576 e29) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] prepare_stats_for_publish reporting purged_snaps [2~1] 2019-12-07T19:34:06.688+0000 7fe2caa6c700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] publish_stats_to_osd 29:86 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] remove_repop repgather(0x558e83dccb40 29'15 rep_tid=26 committed?=1 r=0) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_finish_rmw: Op(1:41df217a:::smithi13140796-39:4 v=29'15 tt=0'0 tid=26 reqid=osd.0.0:26 roll_forward_to=0'0 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={1:41df217a:::smithi13140796-39:4=[],1:41df217a:::smithi13140796-39:head=[]}) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: starting commit on Op(1:41df217a:::smithi13140796-39:4 v=0'0 tt=0'0 tid=27 reqid=osd.0.0:26 roll_forward_to=29'15 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:06.688+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: ExtentCache( ) 2019-12-07T19:34:06.688+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4554 < min_flush_size 524288 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: written: {} 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: op: Op(1:41df217a:::smithi13140796-39:4 v=0'0 tt=0'0 tid=27 reqid=osd.0.0:26 roll_forward_to=29'15 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] try_reads_to_commit: written_set: {} 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417d400 had 1 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84315680 had 1 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84243180 had 1 2019-12-07T19:34:06.688+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4577 < min_flush_size 524288 2019-12-07T19:34:06.688+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4577 < min_flush_size 524288 2019-12-07T19:34:06.688+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWrite(1.2s2 29/28 ECSubWrite(tid=27, reqid=osd.0.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=29'15)) v2 -- 0x558e83cbe800 con 0x558e818d7c00 2019-12-07T19:34:06.688+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x10a192~11e1 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.688+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x10a192 0x10a192~11e1 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xda192 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x192 2019-12-07T19:34:06.688+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWrite(1.2s5 29/28 ECSubWrite(tid=27, reqid=osd.0.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=29'15)) v2 -- 0x558e83cbf000 con 0x558e818d7000 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x373 and padding block with 0xc8d 2019-12-07T19:34:06.688+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWrite(1.2s1 29/28 ECSubWrite(tid=27, reqid=osd.0.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=29'15)) v2 -- 0x558e83cbc000 con 0x558e818fe400 2019-12-07T19:34:06.688+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- MOSDECSubOpWrite(1.2s3 29/28 ECSubWrite(tid=27, reqid=osd.0.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=29'15)) v2 -- 0x558e819d1800 con 0x558e818d6800 2019-12-07T19:34:06.688+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpWrite(1.2s4 29/28 ECSubWrite(tid=27, reqid=osd.0.0:26, at_version=0'0, trim_to=0'0, roll_forward_to=29'15)) v2 -- 0x558e819cf000 con 0x558e818fe000 2019-12-07T19:34:06.688+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.688+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:06.688+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=27'12 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] append_log log((0'0,29'15], crt=27'12) [] 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] rollforward: entry=29'13 (0'0) error 1:41df217a:::smithi13140796-39:head by client.4540.0:249 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] rollforward: entry=29'14 (27'11) delete 1:41df217a:::smithi13140796-39:4 by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] rollforward: entry=29'15 (27'12) delete 1:41df217a:::smithi13140796-39:head by unknown.0.0:0 2019-12-07T19:34:06.671951+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.688+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] append_log approx pg log length = 15 2019-12-07T19:34:06.688+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.688+0000 7fe2caa6c700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.688+0000 7fe2caa6c700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.688+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:06.688+0000 7fe2ca26b700 10 osd.0 29 queue_for_pg_delete on 1.bs3 e 29 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs3 PGDelete(1.bs3 e29) prio 5 cost 1048576 e29) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e84559500 seq 36 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGDelete(1.bs3 e29) prio 5 cost 1048576 e29) queued 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs3 PGDelete(1.bs3 e29) prio 5 cost 1048576 e29) pg 0x558e81928000 2019-12-07T19:34:06.688+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.2s0_head 0#1:41df217a:::smithi13140796-39:4#e onode 0x558e81a03680 txc 0x558e84559500 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x10b373 2019-12-07T19:34:06.688+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.688+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.688+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.688+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 DeleteSome 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.688+0000 7fe2ca26b700 10 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] do_delete_work 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000217968s (0.000000100s flush + 0.000217868s kv commit) 2019-12-07T19:34:06.688+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.688+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) collection_list 1.bs3_head start GHMIN end GHMAX max 30 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195fb00> 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 kv_submitted 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195fb00 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 finishing 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195fb00 onodes 0x558e8417d400,0x558e84315680 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list range 0x837ffffffffffffffdd0000000 to 0x837ffffffffffffffde0000000 and 0x838000000000000001d0000000 to 0x838000000000000001e0000000 start GHMIN 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195fb00 [] 2019-12-07T19:34:06.688+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.688+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.688+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.688+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.688+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=29) prio 5 cost 1048576 e29) queued 2019-12-07T19:34:06.688+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.2s0_head 0#1:41df217a:::smithi13140796-39:4#e 0x0 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list pend 0x837ffffffffffffffde0000000 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list key 0x8380000000000000015000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f95a40) put_ref 0x0~2000 Blob(0x558e83f95a40 blob([0x1c3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e70 loaded (sbid 0x2814 ref_map(0x1c3000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list oid 3#1:d0000000::::head# end GHMAX 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f949a0) put_ref 0x0~2000 Blob(0x558e83f949a0 blob([0x1c5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2e00 loaded (sbid 0x2815 ref_map(0x1c5000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e83eded90) put_ref 0x0~2000 Blob(0x558e83eded90 blob([0x1c7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3180 loaded (sbid 0x2816 ref_map(0x1c7000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f94a10) put_ref 0x0~2000 Blob(0x558e83f94a10 blob([0x1c9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2c40 loaded (sbid 0x2817 ref_map(0x1c9000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list key 0x838000000000000001e000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f95c00) put_ref 0x0~2000 Blob(0x558e83f95c00 blob([0x1cb000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2b60 loaded (sbid 0x2818 ref_map(0x1cb000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e83f94690) put_ref 0x0~2000 Blob(0x558e83f94690 blob([0x1cd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2af0 loaded (sbid 0x2819 ref_map(0x1cd000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) collection_list 1.bs3_head start GHMAX end GHMAX max 30 = 0, ls.size() = 1, next = GHMAX 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e840145b0) put_ref 0x0~2000 Blob(0x558e840145b0 blob([0x1cf000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2380 loaded (sbid 0x281a ref_map(0x1cf000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e840143f0) put_ref 0x0~2000 Blob(0x558e840143f0 blob([0x1d1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd22a0 loaded (sbid 0x281b ref_map(0x1d1000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e84014850) put_ref 0x0~2000 Blob(0x558e84014850 blob([0x1d3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd29a0 loaded (sbid 0x281c ref_map(0x1d3000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] do_delete_work [3#1:d0000000::::head#] 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 osd.0 pg_epoch: 29 pg[1.bs3( v 21'12 (0'0,21'12] lb MIN local-lis/les=18/19 n=3 ec=18/18 lis/c=26/18 les/c/f=27/19/0 sis=28 pruub=13.949732124s) [4,0,2,7,1,5]p4(0) r=-1 lpr=28 DELETING pi=[18,28)/1 crt=21'12 unknown NOTIFY pruub 44.069625289s@ mbc={} ps=[2~1]] do_delete_work finished 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e84014150) put_ref 0x0~2000 Blob(0x558e84014150 blob([0x1d5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd32d0 loaded (sbid 0x281d ref_map(0x1d5000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c41e0 1.bs3_head 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e84014930) put_ref 0x0~2000 Blob(0x558e84014930 blob([0x1d7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd33b0 loaded (sbid 0x281e ref_map(0x1d7000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81795cc0 = 0x558e845c3500 seq 32 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e841a2a10) put_ref 0x0~2000 Blob(0x558e841a2a10 blob([0x1d9000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3490 loaded (sbid 0x281f ref_map(0x1d9000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.bs3_head 3#1:d0000000::::head# onode 0x558e81a02500 txc 0x558e845c3500 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e841a29a0) put_ref 0x0~2000 Blob(0x558e841a29a0 blob([0x1db000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3570 loaded (sbid 0x2820 ref_map(0x1db000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.bs3_head 3#1:d0000000::::head# 0x0 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 bluestore.onode(0x558e81a02500).flush flush done 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6ff10) put_ref 0x0~2000 Blob(0x558e83e6ff10 blob([0x1dd000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3650 loaded (sbid 0x2821 ref_map(0x1dd000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fdc0) put_ref 0x0~2000 Blob(0x558e83e6fdc0 blob([0x1df000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3730 loaded (sbid 0x2822 ref_map(0x1df000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fa40) put_ref 0x0~2000 Blob(0x558e83e6fa40 blob([0x1e1000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3810 loaded (sbid 0x2823 ref_map(0x1e1000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fab0) put_ref 0x0~2000 Blob(0x558e83e6fab0 blob([0x1e3000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd38f0 loaded (sbid 0x2824 ref_map(0x1e3000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fb20) put_ref 0x0~2000 Blob(0x558e83e6fb20 blob([0x1e5000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd39d0 loaded (sbid 0x2825 ref_map(0x1e5000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fc70) put_ref 0x0~2000 Blob(0x558e83e6fc70 blob([0x1e7000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd3ab0 loaded (sbid 0x2826 ref_map(0x1e7000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.blob(0x558e83e6fce0) put_ref 0x0~1000 Blob(0x558e83e6fce0 blob([0x1e9000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83dd3b90 loaded (sbid 0x2827 ref_map(0x1e9000~1000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore.extentmap(0x558e81a03788) dirty_range mark inline shard dirty 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e83f95a40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd2e70 loaded (sbid 0x2814 ref_map(0x1c3000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1c3000~2000] 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1c3000~2000] from SharedBlob(0x558e83dd2e70 loaded (sbid 0x2814 ref_map())) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1c3000~2000 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e83f949a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd2e00 loaded (sbid 0x2815 ref_map(0x1c5000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1c5000~2000] 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1c5000~2000] from SharedBlob(0x558e83dd2e00 loaded (sbid 0x2815 ref_map())) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1c5000~2000 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e83eded90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd3180 loaded (sbid 0x2816 ref_map(0x1c7000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1c7000~2000] 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1c7000~2000] from SharedBlob(0x558e83dd3180 loaded (sbid 0x2816 ref_map())) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1c7000~2000 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e83f94a10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd2c40 loaded (sbid 0x2817 ref_map(0x1c9000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1c9000~2000] 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1c9000~2000] from SharedBlob(0x558e83dd2c40 loaded (sbid 0x2817 ref_map())) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1c9000~2000 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e83f95c00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd2b60 loaded (sbid 0x2818 ref_map(0x1cb000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1cb000~2000] 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1cb000~2000] from SharedBlob(0x558e83dd2b60 loaded (sbid 0x2818 ref_map())) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1cb000~2000 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_omap_clear remove range start: 0x000000000000042e'-' end: 0x000000000000042e'~' 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e83f94690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd2af0 loaded (sbid 0x2819 ref_map(0x1cd000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1cd000~2000] 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1cd000~2000] from SharedBlob(0x558e83dd2af0 loaded (sbid 0x2819 ref_map())) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1cd000~2000 2019-12-07T19:34:06.688+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.bs3_head 3#1:d0000000::::head# = 0 2019-12-07T19:34:06.688+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove_collection 1.bs3_head 2019-12-07T19:34:06.688+0000 7fe2ca26b700 20 bluestore.OnodeSpace(0x558e819c4320 in 0x558e8100e800) map_any 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e840145b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd2380 loaded (sbid 0x281a ref_map(0x1cf000~2000=1)))) 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1cf000~2000] 2019-12-07T19:34:06.688+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1cf000~2000] from SharedBlob(0x558e83dd2380 loaded (sbid 0x281a ref_map())) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1cf000~2000 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558e840143f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd22a0 loaded (sbid 0x281b ref_map(0x1d1000~2000=1)))) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1d1000~2000] 2019-12-07T19:34:06.689+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list range 0x837ffffffffffffffdd0000000 to 0x837ffffffffffffffde0000000 and 0x838000000000000001d0000000 to 0x838000000000000001e0000000 start GHMIN 2019-12-07T19:34:06.689+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list pend 0x837ffffffffffffffde0000000 2019-12-07T19:34:06.689+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list key 0x8380000000000000015000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:06.689+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list oid 3#1:d0000000::::head# end GHMAX 2019-12-07T19:34:06.689+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _collection_list key 0x838000000000000001e000000021213dfffffffffffffffeffffffffffffffff'o' >= GHMAX 2019-12-07T19:34:06.689+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove_collection oid 3#1:d0000000::::head# 2019-12-07T19:34:06.689+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _osr_register_zombie 0x558e81795cc0 1.bs3_head 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1d1000~2000] from SharedBlob(0x558e83dd22a0 loaded (sbid 0x281b ref_map())) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1d1000~2000 2019-12-07T19:34:06.689+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove_collection 1.bs3_head = 0 2019-12-07T19:34:06.689+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845c3500 cost 4267 (1 ios * 4000 + 267 bytes) 2019-12-07T19:34:06.689+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845c3500 onodes shared_blobs 2019-12-07T19:34:06.689+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845c3500 allocated 0x[] released 0x[] 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558e84014850 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd29a0 loaded (sbid 0x281c ref_map(0x1d3000~2000=1)))) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1d3000~2000] 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1d3000~2000] from SharedBlob(0x558e83dd29a0 loaded (sbid 0x281c ref_map())) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1d3000~2000 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558e84014150 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd32d0 loaded (sbid 0x281d ref_map(0x1d5000~2000=1)))) 2019-12-07T19:34:06.689+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c3500 prepare 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1d5000~2000] 2019-12-07T19:34:06.689+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845c3500 2019-12-07T19:34:06.689+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c3500 io_done 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1d5000~2000] from SharedBlob(0x558e83dd32d0 loaded (sbid 0x281d ref_map())) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1d5000~2000 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558e84014930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd33b0 loaded (sbid 0x281e ref_map(0x1d7000~2000=1)))) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1d7000~2000] 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1d7000~2000] from SharedBlob(0x558e83dd33b0 loaded (sbid 0x281e ref_map())) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1d7000~2000 2019-12-07T19:34:06.689+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558e841a2a10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd3490 loaded (sbid 0x281f ref_map(0x1d9000~2000=1)))) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1d9000~2000] 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1d9000~2000] from SharedBlob(0x558e83dd3490 loaded (sbid 0x281f ref_map())) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1d9000~2000 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558e841a29a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd3570 loaded (sbid 0x2820 ref_map(0x1db000~2000=1)))) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1db000~2000] 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1db000~2000] from SharedBlob(0x558e83dd3570 loaded (sbid 0x2820 ref_map())) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1db000~2000 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558e83e6ff10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd3650 loaded (sbid 0x2821 ref_map(0x1dd000~2000=1)))) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1dd000~2000] 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1dd000~2000] from SharedBlob(0x558e83dd3650 loaded (sbid 0x2821 ref_map())) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1dd000~2000 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558e83e6fdc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd3730 loaded (sbid 0x2822 ref_map(0x1df000~2000=1)))) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1df000~2000] 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1df000~2000] from SharedBlob(0x558e83dd3730 loaded (sbid 0x2822 ref_map())) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1df000~2000 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558e83e6fa40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd3810 loaded (sbid 0x2823 ref_map(0x1e1000~2000=1)))) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1e1000~2000] 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1e1000~2000] from SharedBlob(0x558e83dd3810 loaded (sbid 0x2823 ref_map())) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1e1000~2000 2019-12-07T19:34:06.689+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x20000~2000: 0x0~2000 Blob(0x558e83e6fab0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd38f0 loaded (sbid 0x2824 ref_map(0x1e3000~2000=1)))) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1e3000~2000] 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1e3000~2000] from SharedBlob(0x558e83dd38f0 loaded (sbid 0x2824 ref_map())) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1e3000~2000 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x22000~2000: 0x0~2000 Blob(0x558e83e6fb20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd39d0 loaded (sbid 0x2825 ref_map(0x1e5000~2000=1)))) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1e5000~2000] 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1e5000~2000] from SharedBlob(0x558e83dd39d0 loaded (sbid 0x2825 ref_map())) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1e5000~2000 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x24000~2000: 0x0~2000 Blob(0x558e83e6fc70 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd3ab0 loaded (sbid 0x2826 ref_map(0x1e7000~2000=1)))) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1e7000~2000] 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1e7000~2000] from SharedBlob(0x558e83dd3ab0 loaded (sbid 0x2826 ref_map())) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1e7000~2000 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x26000~1000: 0x0~1000 Blob(0x558e83e6fce0 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558e83dd3b90 loaded (sbid 0x2827 ref_map(0x1e9000~1000=1)))) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x1e9000~1000] 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x1e9000~1000] from SharedBlob(0x558e83dd3b90 loaded (sbid 0x2827 ref_map())) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x1e9000~1000 2019-12-07T19:34:06.689+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove gen and maybe_unshared_blobs 0x558e83dd22a0,0x558e83dd2380,0x558e83dd29a0,0x558e83dd2af0,0x558e83dd2b60,0x558e83dd2c40,0x558e83dd2e00,0x558e83dd2e70,0x558e83dd3180,0x558e83dd32d0,0x558e83dd33b0,0x558e83dd3490,0x558e83dd3570,0x558e83dd3650,0x558e83dd3730,0x558e83dd3810,0x558e83dd38f0,0x558e83dd39d0,0x558e83dd3ab0,0x558e83dd3b90 2019-12-07T19:34:06.689+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.2s0_head 0#1:41df217a:::smithi13140796-39:4#e = 0 2019-12-07T19:34:06.689+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.2s0_head 0#1:41df217a:::smithi13140796-39:head#f onode 0x558e83cb3680 txc 0x558e84559500 2019-12-07T19:34:06.689+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.2s0_head 0#1:41df217a:::smithi13140796-39:head#f 0x0 2019-12-07T19:34:06.689+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.2s0_head 0#1:41df217a:::smithi13140796-39:head#f = 0 2019-12-07T19:34:06.689+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.689+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:06.689+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84559500 cost 5587 (1 ios * 4000 + 1587 bytes) 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84559500 onodes shared_blobs 0x558e83dd22a0,0x558e83dd2380,0x558e83dd29a0,0x558e83dd2af0,0x558e83dd2b60,0x558e83dd2c40,0x558e83dd2e00,0x558e83dd2e70,0x558e83dd3180,0x558e83dd32d0,0x558e83dd33b0,0x558e83dd3490,0x558e83dd3570,0x558e83dd3650,0x558e83dd3730,0x558e83dd3810,0x558e83dd38f0,0x558e83dd39d0,0x558e83dd3ab0,0x558e83dd3b90 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281b is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281a is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281c is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2819 is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2818 is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2817 is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2815 is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2814 is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2816 is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281d is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281e is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x281f is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2820 is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2821 is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2822 is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2823 is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2824 is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2825 is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2826 is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2827 is empty 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84559500 allocated 0x[] released 0x[1c3000~27000] 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv release 0x1c3000~27000 2019-12-07T19:34:06.689+0000 7fe2caa6c700 10 freelist release 0x1c3000~27000 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:34:06.689+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559500 prepare 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84559500 2019-12-07T19:34:06.689+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559500 io_done 2019-12-07T19:34:06.689+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=29) prio 5 cost 1048576 e29) pg 0x558e81075000 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] snap_trimmer posting 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 4 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 1 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 bluestore.OmapIteratorImpl(0x558e8400cf80) upper_bound after SNA_1_0000000000000004_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.0_0000000000000001.2' 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 bluestore.OmapIteratorImpl(0x558e8400cf80) valid is at 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.1_0000000000000001.FE9893EA.4.smithi13140796-20..' 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000004_.0_0000000000000001.2) returns 0 SNA_1_0000000000000004_.1_0000000000000001.FE9893EA.4.smithi13140796-20..,buffer::list(len=70, buffer::ptr(0~70 0x558e817e0c60 in raw 0x558e817e0c60 len 70 nref 1) ) 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[4~1] ps=[2~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[2~1]] SnapTrimmer state: adding snap 4 to purged_snaps 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[2~1,4~1]] SnapTrimmer state: purged_snaps now [2~1,4~1], snap_trimq now [] 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e83e73b00 seq 37 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:06.689+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 824 < min_flush_size 524288 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e73b00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e73b00 onodes shared_blobs 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e73b00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73b00 prepare 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e73b00 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73b00 io_done 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[2~1,4~1]] share_pg_info 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02500 had 1 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2368 < min_flush_size 524288 2019-12-07T19:34:06.689+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558e83ebd100 con 0x558e818d7c00 2019-12-07T19:34:06.689+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558e83c9d500 con 0x558e818d7000 2019-12-07T19:34:06.689+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558e83c9f800 con 0x558e818fe400 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 2 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03680 had 1 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3680 had 1 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2391 < min_flush_size 524288 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2391 < min_flush_size 524288 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x10b373~957 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.689+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558e83c9e300 con 0x558e818d6800 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x10b373 0x10b373~957 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xdb373 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x373 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xcca and padding block with 0x336 2019-12-07T19:34:06.689+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 -- 0x558e80ff2300 con 0x558e818fe000 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[2~1,4~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[2~1,4~1]] exit Trimming 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] prepare_stats_for_publish reporting purged_snaps [2~1,4~1] 2019-12-07T19:34:06.689+0000 7fe2c6a64700 15 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] publish_stats_to_osd 29:87 2019-12-07T19:34:06.689+0000 7fe2ca26b700 20 osd.0 29 try_finish_pg_delete 1.bs3 0x558e81928000 2019-12-07T19:34:06.689+0000 7fe2ca26b700 10 osd.0:3._detach_pg 1.bs3 0x558e81928000 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] enter NotTrimming 2019-12-07T19:34:06.689+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:06.689+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:06.689+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] snap_trimmer complete 2019-12-07T19:34:06.689+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x10bcca 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000228823s (0.000000116s flush + 0.000228707s kv commit) 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 snap_mapper.get_snaps 1:15219530:::smithi13140796-21:4 4 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 snap_mapper.clear_snaps 1:15219530:::smithi13140796-21:4 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 snap_mapper.clear_snaps rm OBJ_.5_0000000000000001.8A489AC0.4.smithi13140796-21.. 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 snap_mapper._remove_oid rm SNA_1_0000000000000004_.5_0000000000000001.8A489AC0.4.smithi13140796-21.. 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x10bcca~46d to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x10bcca 0x10bcca~46d to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xdbcca 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xcca 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x137 and padding block with 0xec9 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'25 (0'0,29'25] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=27'23 lcod 27'23 active mbc={} ps=[2~1]] append_log log((0'0,29'25], crt=27'23) [29'26 (27'22) delete 1:15219530:::smithi13140796-21:4 by unknown.0.0:0 2019-12-07T19:34:06.686996+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,29'27 (27'23) delete 1:15219530:::smithi13140796-21:head by unknown.0.0:0 2019-12-07T19:34:06.686996+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'26 (0'0,29'26] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=29'25 crt=27'23 lcod 27'23 active mbc={} ps=[2~1]] add_log_entry 29'26 (27'22) delete 1:15219530:::smithi13140796-21:4 by unknown.0.0:0 2019-12-07T19:34:06.686996+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e845c3500,0x558e84559500> 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c3500 kv_submitted 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845c3500 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=29'25 crt=27'23 lcod 27'23 active mbc={} ps=[2~1]] add_log_entry 29'27 (27'23) delete 1:15219530:::smithi13140796-21:head by unknown.0.0:0 2019-12-07T19:34:06.686996+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c3500 finishing 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845c3500 onodes 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _queue_reap_collection 0x558e819c41e0 1.bs3_head 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=29'25 crt=27'23 lcod 27'23 active mbc={} ps=[2~1]] append_log approx pg log length = 27 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845c3500 done 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81795cc0 q now empty 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845c3500 [] 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=29'25 crt=27'23 lcod 27'23 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish reaping empty zombie osr 0x558e81795cc0 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559500 kv_submitted 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84559500 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559500 finishing 2019-12-07T19:34:06.689+0000 7fe2cba6e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'26, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84559500 onodes 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84559500 done 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e73b00 kv_submitted 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84559500 [1c3000~27000] 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release 0x1c3000~27000 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd3b90) put 0x558e83dd3b90 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3ba8 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e75200 space 0x558e83dd3ba8 0x0~1000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd3ab0) put 0x558e83dd3ab0 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3ac8 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e75170 space 0x558e83dd3ac8 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd39d0) put 0x558e83dd39d0 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x10c137 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e81a0a300 seq 28 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd39e8 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e750e0 space 0x558e83dd39e8 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd38f0) put 0x558e83dd38f0 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3908 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e75050 space 0x558e83dd3908 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd3810) put 0x558e83dd3810 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3828 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e74fc0 space 0x558e83dd3828 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:15219530:::smithi13140796-21:4#1a key 0x858000000000000001152195'0!smithi13140796-21!='0x0000000000000004000000000000001a'o' 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd3730) put 0x558e83dd3730 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3748 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e74f30 space 0x558e83dd3748 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd3650) put 0x558e83dd3650 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3668 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e74ea0 space 0x558e83dd3668 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd3570) put 0x558e83dd3570 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3588 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e74e10 space 0x558e83dd3588 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd3490) put 0x558e83dd3490 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd34a8 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e74d80 space 0x558e83dd34a8 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd33b0) put 0x558e83dd33b0 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd33c8 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e74cf0 space 0x558e83dd33c8 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd32d0) put 0x558e83dd32d0 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd32e8 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e74c60 space 0x558e83dd32e8 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd3180) put 0x558e83dd3180 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3198 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e74b40 space 0x558e83dd3198 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd2e70) put 0x558e83dd2e70 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2e88 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83d114d0 space 0x558e83dd2e88 0x0~1000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e74990 space 0x558e83dd2e88 0x1000~1000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd2e00) put 0x558e83dd2e00 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2e18 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e74900 space 0x558e83dd2e18 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd2c40) put 0x558e83dd2c40 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2c58 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e74a20 space 0x558e83dd2c58 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd2b60) put 0x558e83dd2b60 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2b78 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e74ab0 space 0x558e83dd2b78 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd2af0) put 0x558e83dd2af0 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2b08 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e74bd0 space 0x558e83dd2b08 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd29a0) put 0x558e83dd29a0 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd29b8 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e74870 space 0x558e83dd29b8 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd2380) put 0x558e83dd2380 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2398 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83e747e0 space 0x558e83dd2398 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e83dd22a0) put 0x558e83dd22a0 removing self from set 0x558e819b1390 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd22b8 in 0x558e802195e0) _clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 _rm buffer(0x558e83d11830 space 0x558e83dd22b8 0x0~2000 clean) 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _reap_collections 0x558e819c41e0 1.bs3_head 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore.OnodeSpace(0x558e819c4320 in 0x558e8100e800) map_any 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore.OnodeSpace(0x558e819c4320 in 0x558e8100e800) clear 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _reap_collections 0x558e819c41e0 1.bs3_head done 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _reap_collections all reaped 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.689+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.8s5_head 5#1:15219530:::smithi13140796-21:4# -> 5#1:15219530:::smithi13140796-21:4#1a 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000231344s (0.000000081s flush + 0.000231263s kv commit) 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.8s5_head 5#1:15219530:::smithi13140796-21:4# -> 5#1:15219530:::smithi13140796-21:4#1a = 0 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e73b00> 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73b00 kv_submitted 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e73b00 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73b00 finishing 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e73b00 onodes 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e73b00 done 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:06.689+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e73b00 [] 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:15219530:::smithi13140796-21:head#1b key 0x858000000000000001152195'0!smithi13140796-21!='0xfffffffffffffffe000000000000001b'o' 2019-12-07T19:34:06.689+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.689+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:06.689+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.8s5_head 5#1:15219530:::smithi13140796-21:head# -> 5#1:15219530:::smithi13140796-21:head#1b 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.8s5_head 5#1:15219530:::smithi13140796-21:head# -> 5#1:15219530:::smithi13140796-21:head#1b = 0 2019-12-07T19:34:06.689+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.8A489AC0.4.smithi13140796-21..' <- OBJ_.5_0000000000000001.8A489AC0.4.smithi13140796-21.. 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.689+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.5_0000000000000001.8A489AC0.4.smithi13140796-21..' <- SNA_1_0000000000000004_.5_0000000000000001.8A489AC0.4.smithi13140796-21.. 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.689+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000029.00000000000000000026' <- 0000000029.00000000000000000026 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000029.00000000000000000027' <- 0000000029.00000000000000000027 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a300 cost 6674 (1 ios * 4000 + 2674 bytes) 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a300 onodes 0x558e84315900,0x558e84315b80 shared_blobs 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e84315a08) update 5#1:15219530:::smithi13140796-21:4#1a 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:15219530:::smithi13140796-21:4#1a is 627 (412 bytes onode + 2 bytes spanning blobs + 213 bytes inline extents) 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e84315c88) update 5#1:15219530:::smithi13140796-21:head#1b 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:15219530:::smithi13140796-21:head#1b is 475 (467 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a300 allocated 0x[] released 0x[] 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 prepare 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a300 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 io_done 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=29'25 crt=27'23 lcod 27'23 active mbc={} ps=[2~1]] op_applied version 29'27 2019-12-07T19:34:06.689+0000 7fe2cba6e700 10 osd.0 29 dequeue_op 0x558e8477a2c0 finish 2019-12-07T19:34:06.689+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 29/28 ECSubWriteReply(tid=138, last_complete=29'25, committed=1, applied=1)) v2 -- 0x558e81965400 con 0x558e818d7c00 2019-12-07T19:34:06.689+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3140 < min_flush_size 524288 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84315900 had 1 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84315b80 had 1 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2000 had 1 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:06.689+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84314280 had 1 2019-12-07T19:34:06.689+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 37 ==== MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818a8800 con 0x558e818fe000 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3163 < min_flush_size 524288 2019-12-07T19:34:06.689+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3163 < min_flush_size 524288 2019-12-07T19:34:06.690+0000 7fe2e753b700 15 osd.0 29 enqueue_op 0x558e8477bb80 prio 196 cost 0 latency 0.000019 epoch 29 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.690+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.690+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.690+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x10c137~c5b to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.690+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x10c137 0x10c137~c5b to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.690+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.690+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xdc137 2019-12-07T19:34:06.690+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x137 2019-12-07T19:34:06.690+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.690+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd92 and padding block with 0x26e 2019-12-07T19:34:06.690+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 120 ==== MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818ef600 con 0x558e818fe400 2019-12-07T19:34:06.690+0000 7fe2e6d3a700 15 osd.0 29 enqueue_op 0x558e83dd8000 prio 196 cost 0 latency 0.000016 epoch 29 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.690+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.690+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x10cd92 2019-12-07T19:34:06.690+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.690+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.690+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.690+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.690+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000249164s (0.000000103s flush + 0.000249061s kv commit) 2019-12-07T19:34:06.690+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0a300> 2019-12-07T19:34:06.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.690+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 kv_submitted 2019-12-07T19:34:06.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a300 2019-12-07T19:34:06.690+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 finishing 2019-12-07T19:34:06.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a300 onodes 0x558e84315900,0x558e84315b80 2019-12-07T19:34:06.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a300 done 2019-12-07T19:34:06.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:06.690+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a300 [] 2019-12-07T19:34:06.690+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.690+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.690+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.694+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 29/28 ECSubWriteReply(tid=140, last_complete=29'27, committed=1, applied=1)) v2 -- 0x558e84196000 con 0x558e818d7c00 2019-12-07T19:34:06.694+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.690+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.690+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 57 ==== MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e818ef200 con 0x558e818d6800 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.691+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 118 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83ebd100 con 0x558e818d7000 2019-12-07T19:34:06.694+0000 7fe2e753b700 15 osd.0 29 enqueue_op 0x558e8477bce0 prio 196 cost 0 latency 0.004225 epoch 29 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.694+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.694+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.1s1 epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:06.694+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e8477bb80 prio 196 cost 0 latency 0.004840 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] 2019-12-07T19:34:06.694+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.694+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e8477bb80 finish 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.694+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 119 ==== MOSDECSubOpWrite(1.1s1 29/28 ECSubWrite(tid=88, reqid=osd.3.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=29'9)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83cbe800 con 0x558e818d7000 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.694+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e8477a420 prio 127 cost 0 latency 0.000021 epoch 29 MOSDECSubOpWrite(1.1s1 29/28 ECSubWrite(tid=88, reqid=osd.3.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=29'9)) v2 2019-12-07T19:34:06.694+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 29/28 ECSubWrite(tid=88, reqid=osd.3.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=29'9)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.694+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e83dd8000 prio 196 cost 0 latency 0.004780 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] 2019-12-07T19:34:06.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.694+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e83dd8000 finish 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.694+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 120 ==== MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e81965400 con 0x558e818d7000 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.694+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e8477a580 prio 196 cost 0 latency 0.000010 epoch 29 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.694+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.694+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e8477bce0 prio 196 cost 0 latency 0.004323 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] 2019-12-07T19:34:06.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.694+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e8477bce0 finish 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.694+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e8477a580 prio 196 cost 0 latency 0.000037 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] 2019-12-07T19:34:06.694+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 121 ==== MOSDECSubOpWrite(1.fs1 29/28 ECSubWrite(tid=89, reqid=osd.3.0:89, at_version=29'12, trim_to=0'0, roll_forward_to=0'0)) v2 ==== 2656+0+0 (unknown 0 0 0) 0x558e83cbf000 con 0x558e818d7000 2019-12-07T19:34:06.694+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e8477a6e0 prio 127 cost 0 latency 0.000027 epoch 29 MOSDECSubOpWrite(1.fs1 29/28 ECSubWrite(tid=89, reqid=osd.3.0:89, at_version=29'12, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.694+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.694+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 29/28 ECSubWrite(tid=89, reqid=osd.3.0:89, at_version=29'12, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.694+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e8477a580 finish 2019-12-07T19:34:06.694+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.692+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.694+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.692+0000 7fe2ca26b700 10 bluestore.OnodeSpace(0x558e819c4320 in 0x558e8100e800) clear 2019-12-07T19:34:06.694+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:06.695+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 218 ==== MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 ==== 95+0+0 (unknown 0 0 0) 0x558e81238c00 con 0x558e818d7c00 2019-12-07T19:34:06.695+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e8477a840 prio 196 cost 0 latency 0.000012 epoch 29 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.695+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) 2019-12-07T19:34:06.695+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 219 ==== MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=144, reqid=osd.1.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=29'13)) v2 ==== 909+0+0 (unknown 0 0 0) 0x558e83aca000 con 0x558e818d7c00 2019-12-07T19:34:06.695+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e8477a9a0 prio 127 cost 0 latency 0.000016 epoch 29 MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=144, reqid=osd.1.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=29'13)) v2 2019-12-07T19:34:06.695+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=144, reqid=osd.1.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=29'13)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.695+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 220 ==== MOSDECSubOpRead(1.9s3 29/28 ECSubRead(tid=145, to_read={1:9da56077:::smithi13140796-13:head=0,4096,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8417db80 con 0x558e818d7c00 2019-12-07T19:34:06.695+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e8477ac60 prio 127 cost 0 latency 0.000015 epoch 29 MOSDECSubOpRead(1.9s3 29/28 ECSubRead(tid=145, to_read={1:9da56077:::smithi13140796-13:head=0,4096,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:06.695+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 29/28 ECSubRead(tid=145, to_read={1:9da56077:::smithi13140796-13:head=0,4096,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e29) 2019-12-07T19:34:06.695+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 221 ==== pg_info2(1.9s3 1.9s3 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83ebd100 con 0x558e818d7c00 2019-12-07T19:34:06.695+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.9s3 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:06.695+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.695+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.695+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) queued 2019-12-07T19:34:06.695+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.695+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2) prio 196 cost 0 e29) pg 0x558e81075000 2019-12-07T19:34:06.695+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e8477a840 prio 196 cost 0 latency 0.000120 MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] 2019-12-07T19:34:06.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] _handle_message: MOSDECSubOpWriteReply(1.2s0 29/28 ECSubWriteReply(tid=27, last_complete=29'15, committed=1, applied=1)) v2 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.695+0000 7fe2caa6c700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] try_finish_rmw: Op(1:41df217a:::smithi13140796-39:4 v=0'0 tt=0'0 tid=27 reqid=osd.0.0:26 roll_forward_to=29'15 temp_added= temp_cleared= pending_read={} remote_read={} remote_read_result={} pending_apply= pending_commit= plan.to_read={} plan.will_write={}) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e29) pg 0x558e819e4000 2019-12-07T19:34:06.695+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] try_finish_rmw: ExtentCache( ) 2019-12-07T19:34:06.695+0000 7fe2caa6c700 20 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] try_finish_rmw: clearing pipeline_state CACHE_VALID 2019-12-07T19:34:06.695+0000 7fe2caa6c700 10 osd.0 29 dequeue_op 0x558e8477a840 finish 2019-12-07T19:34:06.695+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=27'7 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=27'7 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e83c56f00 seq 27 2019-12-07T19:34:06.695+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56f00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56f00 onodes shared_blobs 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56f00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 prepare 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56f00 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 io_done 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e29) pg 0x558e8192b000 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x10cd92~46d to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x10cd92 0x10cd92~46d to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xdcd92 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd92 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x1ff and padding block with 0xe01 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83c56600 seq 30 2019-12-07T19:34:06.695+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56600 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56600 onodes shared_blobs 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56600 allocated 0x[] released 0x[] 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 prepare 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56600 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 io_done 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 29/28 ECSubWrite(tid=88, reqid=osd.3.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=29'9)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x10d1ff 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpWrite(1.1s1 29/28 ECSubWrite(tid=88, reqid=osd.3.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=29'9)) v2) prio 127 cost 0 e29) pg 0x558e819e4000 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000090318s (0.000000145s flush + 0.000090173s kv commit) 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 osd.0 29 dequeue_op 0x558e8477a420 prio 127 cost 0 latency 0.000507 MOSDECSubOpWrite(1.1s1 29/28 ECSubWrite(tid=88, reqid=osd.3.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=29'9)) v2 pg pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=27'7 active mbc={} ps=[2~1,4~1]] 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56f00> 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 kv_submitted 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56f00 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 finishing 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56f00 onodes 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=27'7 active mbc={} ps=[2~1,4~1]] _handle_message: MOSDECSubOpWrite(1.1s1 29/28 ECSubWrite(tid=88, reqid=osd.3.0:86, at_version=0'0, trim_to=0'0, roll_forward_to=29'9)) v2 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56f00 done 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56f00 [] 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x10d1ff~46d to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=27'7 active mbc={} ps=[2~1,4~1]] append_log log((0'0,29'9], crt=27'7) [] 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x10d1ff 0x10d1ff~46d to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xdd1ff 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x1ff 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x66c and padding block with 0x994 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=144, reqid=osd.1.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=29'13)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=144, reqid=osd.1.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=29'13)) v2) prio 127 cost 0 e29) pg 0x558e8192b000 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 pg_epoch: 29 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] rollforward: entry=29'8 (27'4) delete 1:891f0d13:::smithi13140796-14:4 by unknown.0.0:0 2019-12-07T19:34:06.673165+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 osd.0 29 dequeue_op 0x558e8477a9a0 prio 127 cost 0 latency 0.000343 MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=144, reqid=osd.1.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=29'13)) v2 pg pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 pg_epoch: 29 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] rollforward: entry=29'9 (27'7) modify 1:891f0d13:::smithi13140796-14:head by unknown.0.0:0 2019-12-07T19:34:06.673165+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] _handle_message: MOSDECSubOpWrite(1.9s3 29/28 ECSubWrite(tid=144, reqid=osd.1.0:139, at_version=0'0, trim_to=0'0, roll_forward_to=29'13)) v2 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] append_log log((0'0,29'13], crt=27'8) [] 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] append_log approx pg log length = 9 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] rollforward: entry=29'9 (21'6) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:244 2019-12-07T19:34:05.664687+0000 0 ObjectCleanRegions clean_offsets: [0~212992,966656~18446744073708584959], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.695+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] rollforward: entry=29'10 (29'9) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:245 2019-12-07T19:34:05.667234+0000 0 ObjectCleanRegions clean_offsets: [0~966656,1343488~18446744073708208127], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] rollforward: entry=29'11 (29'10) modify 1:9da56077:::smithi13140796-13:head by client.4540.0:246 2019-12-07T19:34:05.667270+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e8195e900 seq 28 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] rollforward: entry=29'12 (27'7) delete 1:9694a0e5:::smithi13140796-3:4 by unknown.0.0:0 2019-12-07T19:34:06.683044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.695+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4#8 onode 0x558e8417ca00 txc 0x558e8195e900 2019-12-07T19:34:06.695+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4#8 0x0 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x10d66c 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore.blob(0x558e83e6d5e0) put_ref 0x0~2000 Blob(0x558e83e6d5e0 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] rollforward: entry=29'13 (27'8) modify 1:9694a0e5:::smithi13140796-3:head by unknown.0.0:0 2019-12-07T19:34:06.683044+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore.blob(0x558e83e6d570) put_ref 0x0~2000 Blob(0x558e83e6d570 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore.blob(0x558e83f95d50) put_ref 0x0~2000 Blob(0x558e83f95d50 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] append_log approx pg log length = 13 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000121080s (0.000000069s flush + 0.000121011s kv commit) 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore.blob(0x558e84014070) put_ref 0x0~2000 Blob(0x558e84014070 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore.blob(0x558e84014380) put_ref 0x0~2000 Blob(0x558e84014380 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore.blob(0x558e840140e0) put_ref 0x0~2000 Blob(0x558e840140e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore.blob(0x558e84014230) put_ref 0x0~2000 Blob(0x558e84014230 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore.extentmap(0x558e8417cb08) dirty_range mark inline shard dirty 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e83e6d5e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=2)))) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x5c000~2000] 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=1))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e83e6d570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=2)))) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x5e000~2000] 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e819a0c00 seq 31 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=1))) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e83f95d50 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=2)))) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x60000~2000] 2019-12-07T19:34:06.695+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4#c onode 0x558e8402f180 txc 0x558e819a0c00 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=1))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4#c 0x0 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e84014070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=2)))) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x62000~2000] 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore.blob(0x558e83d03420) put_ref 0x0~2000 Blob(0x558e83d03420 blob([0x37000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1)))) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=1))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore.blob(0x558e83d03260) put_ref 0x0~1000 Blob(0x558e83d03260 blob([0x39000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=1)))) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e84014380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=2)))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore.extentmap(0x558e8402f288) dirty_range mark inline shard dirty 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x64000~2000] 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=1))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e83d03420 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map(0x37000~2000=1)))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x37000~2000] 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e840140e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=2)))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x37000~2000] from SharedBlob(0x558e817e8d20 loaded (sbid 0x2812 ref_map())) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x66000~2000] 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x37000~2000 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=1))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~1000: 0x0~1000 Blob(0x558e83d03260 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map(0x39000~1000=1)))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x39000~1000] 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x39000~1000] from SharedBlob(0x558e8194f3b0 loaded (sbid 0x2813 ref_map())) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e84014230 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=2)))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x39000~1000 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x68000~2000] 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=1))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove gen and maybe_unshared_blobs 0x558e817e8d20,0x558e8194f3b0 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.9s3_head 3#1:9694a0e5:::smithi13140796-3:4#c = 0 2019-12-07T19:34:06.695+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove gen and maybe_unshared_blobs 0x558e817e9650,0x558e817e96c0,0x558e817e9b90,0x558e817e9c70,0x558e83c64070,0x558e83c657a0,0x558e83c658f0 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.1s1_head 1#1:891f0d13:::smithi13140796-14:4#8 = 0 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0c00 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:06.695+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0c00 onodes shared_blobs 0x558e817e8d20,0x558e8194f3b0 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2812 is empty 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2813 is empty 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0c00 allocated 0x[] released 0x[37000~3000] 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv release 0x37000~3000 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 freelist release 0x37000~3000 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e900 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 prepare 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0c00 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e900 onodes shared_blobs 0x558e817e9650,0x558e817e96c0,0x558e817e9b90,0x558e817e9c70,0x558e83c64070,0x558e83c657a0,0x558e83c658f0 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 io_done 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2807 is 11 SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=1))) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2809 is 11 SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=1))) 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 osd.0 29 dequeue_op 0x558e8477a9a0 finish 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2805 is 11 SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=1))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x280b is 11 SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=1))) 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1273 < min_flush_size 524288 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2801 is 11 SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=1))) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2802 is 11 SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=1))) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 29/28 ECSubRead(tid=145, to_read={1:9da56077:::smithi13140796-13:head=0,4096,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2803 is 11 SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=1))) 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e900 allocated 0x[] released 0x[] 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpRead(1.9s3 29/28 ECSubRead(tid=145, to_read={1:9da56077:::smithi13140796-13:head=0,4096,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e29) pg 0x558e8192b000 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 prepare 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e900 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 io_done 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:06.695+0000 7fe2c7265700 10 osd.0 29 dequeue_op 0x558e8477a420 finish 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402f180 had 1 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1296 < min_flush_size 524288 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1296 < min_flush_size 524288 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 osd.0 29 dequeue_op 0x558e8477ac60 prio 127 cost 0 latency 0.000521 MOSDECSubOpRead(1.9s3 29/28 ECSubRead(tid=145, to_read={1:9da56077:::smithi13140796-13:head=0,4096,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x10d66c~510 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] _handle_message: MOSDECSubOpRead(1.9s3 29/28 ECSubRead(tid=145, to_read={1:9da56077:::smithi13140796-13:head=0,4096,0}, subchunks={1:9da56077:::smithi13140796-13:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x10d66c 0x10d66c~510 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xdd66c 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x66c 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.695+0000 7fe2cb26d700 25 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb7c and padding block with 0x484 2019-12-07T19:34:06.695+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x0~1000 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1000 size 0x52000 (335872) 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e70 blob([0x4f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64d90 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 0x0~1000 = 4096 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] handle_sub_read read request=4096 r=4096 len=4096 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] get_hash_info: Getting attr on 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] get_hash_info: not in cache 1:9da56077:::smithi13140796-13:head 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] get_hash_info: found on disk, size 335872 2019-12-07T19:34:06.695+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.9s3_head 3#1:9da56077:::smithi13140796-13:head# hinfo_key = 0 2019-12-07T19:34:06.695+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpReadReply(1.9s0 29/28 ECSubReadReply(tid=145, attrs_read=0)) v2 -- 0x558e83c72c80 con 0x558e818d7c00 2019-12-07T19:34:06.695+0000 7fe2cb26d700 10 osd.0 29 dequeue_op 0x558e8477ac60 finish 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56600> 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 kv_submitted 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56600 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56600 finishing 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56600 onodes 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56600 done 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0c00 kv_submitted 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56600 [] 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x10db7c 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000139424s (0.000000057s flush + 0.000139367s kv commit) 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a0c00> 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 kv_submitted 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0c00 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 finishing 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0c00 onodes 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0c00 done 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1389 < min_flush_size 524288 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0c00 [37000~3000] 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release 0x37000~3000 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e8194f3b0) put 0x558e8194f3b0 removing self from set 0x558e819b0c10 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194f3c8 in 0x558e80219a40) _clear 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 _rm buffer(0x558e81a47cb0 space 0x558e8194f3c8 0x0~1000 clean) 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e817e8d20) put 0x558e817e8d20 removing self from set 0x558e819b0c10 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e817e8d38 in 0x558e80219a40) _clear 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 _rm buffer(0x558e81170f30 space 0x558e817e8d38 0x0~1000 clean) 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 _rm buffer(0x558e81a47c20 space 0x558e817e8d38 0x1000~1000 clean) 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417ca00 had 1 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1412 < min_flush_size 524288 2019-12-07T19:34:06.695+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1412 < min_flush_size 524288 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 29/28 ECSubWrite(tid=89, reqid=osd.3.0:89, at_version=29'12, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x10db7c~584 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x10db7c 0x10db7c~584 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xddb7c 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb7c 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.695+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 29/28 ECSubWrite(tid=89, reqid=osd.3.0:89, at_version=29'12, trim_to=0'0, roll_forward_to=0'0)) v2) prio 127 cost 0 e29) pg 0x558e81a9b000 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x100 and padding block with 0xf00 2019-12-07T19:34:06.695+0000 7fe2c8267700 10 osd.0 29 dequeue_op 0x558e8477a6e0 prio 127 cost 0 latency 0.000867 MOSDECSubOpWrite(1.fs1 29/28 ECSubWrite(tid=89, reqid=osd.3.0:89, at_version=29'12, trim_to=0'0, roll_forward_to=0'0)) v2 pg pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.695+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.fs1 29/28 ECSubWrite(tid=89, reqid=osd.3.0:89, at_version=29'12, trim_to=0'0, roll_forward_to=0'0)) v2 2019-12-07T19:34:06.695+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 29/28 ECSubWriteReply(tid=144, last_complete=29'13, committed=1, applied=1)) v2 -- 0x558e818ef000 con 0x558e818d7c00 2019-12-07T19:34:06.695+0000 7fe2c8267700 20 snap_mapper.remove_oid 1:f7919c75:::smithi13140796-20:4 2019-12-07T19:34:06.695+0000 7fe2c8267700 20 snap_mapper._remove_oid 1:f7919c75:::smithi13140796-20:4 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.695+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.695+0000 7fe2c8267700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:06.695+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.695+0000 7fe2c8267700 20 snap_mapper.get_snaps 1:f7919c75:::smithi13140796-20:4 4 2019-12-07T19:34:06.695+0000 7fe2c8267700 20 snap_mapper.clear_snaps 1:f7919c75:::smithi13140796-20:4 2019-12-07T19:34:06.695+0000 7fe2c8267700 20 snap_mapper.clear_snaps rm OBJ_.1_0000000000000001.FE9893EA.4.smithi13140796-20.. 2019-12-07T19:34:06.695+0000 7fe2c8267700 20 snap_mapper._remove_oid rm SNA_1_0000000000000004_.1_0000000000000001.FE9893EA.4.smithi13140796-20.. 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x10e100 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.695+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000129299s (0.000000057s flush + 0.000129242s kv commit) 2019-12-07T19:34:06.695+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] append_log log((0'0,27'10], crt=27'10) [29'11 (27'7) delete 1:f7919c75:::smithi13140796-20:4 by unknown.0.0:0 2019-12-07T19:34:06.691839+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,29'12 (27'10) modify 1:f7919c75:::smithi13140796-20:head by unknown.0.0:0 2019-12-07T19:34:06.691839+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.695+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e900> 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 kv_submitted 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e900 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 finishing 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e900 onodes 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e900 done 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:06.695+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e900 [] 2019-12-07T19:34:06.695+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.695+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 29'11 (0'0,29'11] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 lua=27'10 crt=27'10 lcod 27'10 active mbc={} ps=[2~1]] add_log_entry 29'11 (27'7) delete 1:f7919c75:::smithi13140796-20:4 by unknown.0.0:0 2019-12-07T19:34:06.691839+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.695+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.695+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.1s0 29/28 ECSubWriteReply(tid=88, last_complete=29'9, committed=1, applied=1)) v2 -- 0x558e83fef600 con 0x558e818d7000 2019-12-07T19:34:06.695+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 lua=27'10 crt=27'10 lcod 27'10 active mbc={} ps=[2~1]] add_log_entry 29'12 (27'10) modify 1:f7919c75:::smithi13140796-20:head by unknown.0.0:0 2019-12-07T19:34:06.691839+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.695+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:06.695+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 lua=27'10 crt=27'10 lcod 27'10 active mbc={} ps=[2~1]] append_log approx pg log length = 12 2019-12-07T19:34:06.695+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 lua=27'10 crt=27'10 lcod 27'10 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.695+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.695+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.695+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:06.695+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e81a0bb00 seq 32 2019-12-07T19:34:06.695+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) get_onode oid 1#1:f7919c75:::smithi13140796-20:4#b key 0x818000000000000001f7919c'u!smithi13140796-20!='0x0000000000000004000000000000000b'o' 2019-12-07T19:34:06.695+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.fs1_head 0x558e8122c000) r -2 v.len 0 2019-12-07T19:34:06.695+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# -> 1#1:f7919c75:::smithi13140796-20:4#b 2019-12-07T19:34:06.695+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4# -> 1#1:f7919c75:::smithi13140796-20:4#b = 0 2019-12-07T19:34:06.695+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 2 keys 2019-12-07T19:34:06.696+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# 2 keys = 0 2019-12-07T19:34:06.696+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.696+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.fs1_head 1#1:f7919c75:::smithi13140796-20:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.696+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.FE9893EA.4.smithi13140796-20..' <- OBJ_.1_0000000000000001.FE9893EA.4.smithi13140796-20.. 2019-12-07T19:34:06.696+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.696+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.1_0000000000000001.FE9893EA.4.smithi13140796-20..' <- SNA_1_0000000000000004_.1_0000000000000001.FE9893EA.4.smithi13140796-20.. 2019-12-07T19:34:06.696+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.696+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.0000000029.00000000000000000011' <- 0000000029.00000000000000000011 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.0000000029.00000000000000000012' <- 0000000029.00000000000000000012 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.696+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:06.696+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0bb00 cost 7531 (1 ios * 4000 + 3531 bytes) 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0bb00 onodes 0x558e83c72280,0x558e83c72500 shared_blobs 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83c72388) update 1#1:f7919c75:::smithi13140796-20:4#b 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:f7919c75:::smithi13140796-20:4#b is 691 (412 bytes onode + 2 bytes spanning blobs + 277 bytes inline extents) 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83c72608) update 1#1:f7919c75:::smithi13140796-20:head# 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:f7919c75:::smithi13140796-20:head# is 1383 (485 bytes onode + 2 bytes spanning blobs + 896 bytes inline extents) 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:06.696+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 prepare 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0bb00 2019-12-07T19:34:06.696+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 io_done 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.696+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 lua=27'10 crt=27'10 lcod 27'10 active mbc={} ps=[2~1]] op_applied version 29'12 2019-12-07T19:34:06.696+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4519 < min_flush_size 524288 2019-12-07T19:34:06.696+0000 7fe2c8267700 10 osd.0 29 dequeue_op 0x558e8477a6e0 finish 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.696+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72280 had 1 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72500 had 1 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402e280 had 1 2019-12-07T19:34:06.696+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4542 < min_flush_size 524288 2019-12-07T19:34:06.696+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4542 < min_flush_size 524288 2019-12-07T19:34:06.696+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.696+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x10e100~11be to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.696+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x10e100 0x10e100~11be to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xde100 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x100 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x2be and padding block with 0xd42 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x10f2be 2019-12-07T19:34:06.696+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.696+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.696+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000153454s (0.000000077s flush + 0.000153377s kv commit) 2019-12-07T19:34:06.696+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0bb00> 2019-12-07T19:34:06.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.696+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 kv_submitted 2019-12-07T19:34:06.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0bb00 2019-12-07T19:34:06.696+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 finishing 2019-12-07T19:34:06.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0bb00 onodes 0x558e83c72280,0x558e83c72500 2019-12-07T19:34:06.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0bb00 done 2019-12-07T19:34:06.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:06.696+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0bb00 [] 2019-12-07T19:34:06.696+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.696+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.696+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 29/28 ECSubWriteReply(tid=89, last_complete=29'12, committed=1, applied=1)) v2 -- 0x558e84028c00 con 0x558e818d7000 2019-12-07T19:34:06.696+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.696+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.696+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 222 ==== MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=146, reqid=osd.1.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=29'27)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83acb000 con 0x558e818d7c00 2019-12-07T19:34:06.696+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e8477adc0 prio 127 cost 0 latency 0.000023 epoch 29 MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=146, reqid=osd.1.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=29'27)) v2 2019-12-07T19:34:06.696+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=146, reqid=osd.1.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=29'27)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=146, reqid=osd.1.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=29'27)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=146, reqid=osd.1.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=29'27)) v2) prio 127 cost 0 e29) pg 0x558e841a4000 2019-12-07T19:34:06.696+0000 7fe2c7a66700 10 osd.0 29 dequeue_op 0x558e8477adc0 prio 127 cost 0 latency 0.000092 MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=146, reqid=osd.1.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=29'27)) v2 pg pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.696+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=146, reqid=osd.1.0:140, at_version=0'0, trim_to=0'0, roll_forward_to=29'27)) v2 2019-12-07T19:34:06.696+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=27'23 active mbc={} ps=[2~1]] append_log log((0'0,29'27], crt=27'23) [] 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 osd.0 pg_epoch: 29 pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'27 active mbc={} ps=[2~1]] rollforward: entry=29'24 (27'15) delete 1:10779340:::smithi13140796-32:4 by unknown.0.0:0 2019-12-07T19:34:06.682769+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 osd.0 pg_epoch: 29 pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'27 active mbc={} ps=[2~1]] rollforward: entry=29'25 (27'18) modify 1:10779340:::smithi13140796-32:head by unknown.0.0:0 2019-12-07T19:34:06.682769+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 osd.0 pg_epoch: 29 pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'27 active mbc={} ps=[2~1]] rollforward: entry=29'26 (27'22) delete 1:15219530:::smithi13140796-21:4 by unknown.0.0:0 2019-12-07T19:34:06.686996+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 osd.0 pg_epoch: 29 pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'27 active mbc={} ps=[2~1]] rollforward: entry=29'27 (27'23) delete 1:15219530:::smithi13140796-21:head by unknown.0.0:0 2019-12-07T19:34:06.686996+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.696+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'27 active mbc={} ps=[2~1]] append_log approx pg log length = 27 2019-12-07T19:34:06.696+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'27 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.696+0000 7fe2c7a66700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.696+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.696+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e83d4cc00 seq 29 2019-12-07T19:34:06.696+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.8s5_head 5#1:10779340:::smithi13140796-32:4#18 onode 0x558e8417d400 txc 0x558e83d4cc00 2019-12-07T19:34:06.696+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.8s5_head 5#1:10779340:::smithi13140796-32:4#18 0x0 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f61030) put_ref 0x0~2000 Blob(0x558e83f61030 blob([0x4ba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f61f10 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e83ed4070) put_ref 0x0~2000 Blob(0x558e83ed4070 blob([0x4bc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464db90 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464dc00) put_ref 0x0~2000 Blob(0x558e8464dc00 blob([0x4be000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464dc70 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464dce0) put_ref 0x0~2000 Blob(0x558e8464dce0 blob([0x4c0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464dd50 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464ddc0) put_ref 0x0~2000 Blob(0x558e8464ddc0 blob([0x4c2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464de30 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464dea0) put_ref 0x0~2000 Blob(0x558e8464dea0 blob([0x4c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464df10 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464df80) put_ref 0x0~2000 Blob(0x558e8464df80 blob([0x4c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a000 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a070) put_ref 0x0~2000 Blob(0x558e8456a070 blob([0x4c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a0e0 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a150) put_ref 0x0~2000 Blob(0x558e8456a150 blob([0x4ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a1c0 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a230) put_ref 0x0~2000 Blob(0x558e8456a230 blob([0x4cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a2a0 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a310) put_ref 0x0~2000 Blob(0x558e8456a310 blob([0x4ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a380 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a3f0) put_ref 0x0~2000 Blob(0x558e8456a3f0 blob([0x4d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a460 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a4d0) put_ref 0x0~2000 Blob(0x558e8456a4d0 blob([0x4d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a540 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a5b0) put_ref 0x0~2000 Blob(0x558e8456a5b0 blob([0x4d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a620 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a690) put_ref 0x0~2000 Blob(0x558e8456a690 blob([0x4d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a700 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a770) put_ref 0x0~2000 Blob(0x558e8456a770 blob([0x4d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a7e0 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e8417d508) dirty_range mark inline shard dirty 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e83f61030 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f61f10 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4ba000~2000] 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f61030) discard_unallocated 0x0~2000 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f61f28 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4ba000~2000 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e83f61f10) put 0x558e83f61f10 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f61f28 in 0x558e80219960) _clear 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e83ed4070 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8464db90 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4bc000~2000] 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e83ed4070) discard_unallocated 0x0~2000 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464dba8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4bc000~2000 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8464db90) put 0x558e8464db90 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464dba8 in 0x558e80219960) _clear 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e8464dc00 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8464dc70 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4be000~2000] 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464dc00) discard_unallocated 0x0~2000 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464dc88 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4be000~2000 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8464dc70) put 0x558e8464dc70 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464dc88 in 0x558e80219960) _clear 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e8464dce0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8464dd50 sbid 0x0)) 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4c0000~2000] 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464dce0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464dd68 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4c0000~2000 2019-12-07T19:34:06.696+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8464dd50) put 0x558e8464dd50 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464dd68 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e8464ddc0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8464de30 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4c2000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464ddc0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464de48 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4c2000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8464de30) put 0x558e8464de30 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464de48 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e8464dea0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8464df10 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4c4000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464dea0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464df28 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4c4000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8464df10) put 0x558e8464df10 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8464df28 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e8464df80 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8456a000 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4c6000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e8464df80) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a018 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4c6000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8456a000) put 0x558e8456a000 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a018 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558e8456a070 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8456a0e0 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4c8000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a070) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a0f8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4c8000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8456a0e0) put 0x558e8456a0e0 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a0f8 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558e8456a150 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8456a1c0 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4ca000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a150) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a1d8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4ca000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8456a1c0) put 0x558e8456a1c0 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a1d8 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558e8456a230 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8456a2a0 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4cc000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a230) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a2b8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4cc000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8456a2a0) put 0x558e8456a2a0 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a2b8 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558e8456a310 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8456a380 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4ce000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a310) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a398 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4ce000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8456a380) put 0x558e8456a380 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a398 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558e8456a3f0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8456a460 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4d0000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a3f0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a478 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4d0000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8456a460) put 0x558e8456a460 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a478 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558e8456a4d0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8456a540 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4d2000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a4d0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a558 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4d2000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8456a540) put 0x558e8456a540 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a558 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558e8456a5b0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8456a620 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4d4000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a5b0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a638 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4d4000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8456a620) put 0x558e8456a620 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a638 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1c000~2000: 0x0~2000 Blob(0x558e8456a690 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8456a700 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4d6000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a690) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a718 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4d6000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8456a700) put 0x558e8456a700 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a718 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1e000~2000: 0x0~2000 Blob(0x558e8456a770 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8456a7e0 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x4d8000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a770) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a7f8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x4d8000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e8456a7e0) put 0x558e8456a7e0 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a7f8 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.8s5_head 5#1:10779340:::smithi13140796-32:4#18 = 0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.8s5_head 5#1:15219530:::smithi13140796-21:4#1a onode 0x558e84315900 txc 0x558e83d4cc00 2019-12-07T19:34:06.697+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.8s5_head 5#1:15219530:::smithi13140796-21:4#1a 0x0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f617a0) put_ref 0x0~2000 Blob(0x558e83f617a0 blob([0x56b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5b420 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f61490) put_ref 0x0~2000 Blob(0x558e83f61490 blob([0x56d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84548540 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e845485b0) put_ref 0x0~2000 Blob(0x558e845485b0 blob([0x56f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84548620 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548690) put_ref 0x0~2000 Blob(0x558e84548690 blob([0x571000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84548700 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548770) put_ref 0x0~2000 Blob(0x558e84548770 blob([0x573000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e845487e0 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548850) put_ref 0x0~2000 Blob(0x558e84548850 blob([0x575000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e845488c0 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548930) put_ref 0x0~2000 Blob(0x558e84548930 blob([0x577000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e845489a0 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548a10) put_ref 0x0~2000 Blob(0x558e84548a10 blob([0x579000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84548a80 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548af0) put_ref 0x0~2000 Blob(0x558e84548af0 blob([0x57b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84548b60 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548bd0) put_ref 0x0~2000 Blob(0x558e84548bd0 blob([0x57d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84548c40 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548cb0) put_ref 0x0~1000 Blob(0x558e84548cb0 blob([0x57f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e84548d20 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e84315a08) dirty_range mark inline shard dirty 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e83f617a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f5b420 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x56b000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f617a0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f5b438 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x56b000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e83f5b420) put 0x558e83f5b420 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f5b438 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e83f61490 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84548540 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x56d000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f61490) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548558 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x56d000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e84548540) put 0x558e84548540 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548558 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e845485b0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84548620 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x56f000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e845485b0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548638 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x56f000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e84548620) put 0x558e84548620 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548638 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e84548690 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84548700 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x571000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548690) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548718 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x571000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e84548700) put 0x558e84548700 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548718 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e84548770 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e845487e0 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x573000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548770) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e845487f8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x573000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e845487e0) put 0x558e845487e0 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e845487f8 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e84548850 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e845488c0 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x575000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548850) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e845488d8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x575000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e845488c0) put 0x558e845488c0 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e845488d8 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e84548930 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e845489a0 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x577000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548930) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e845489b8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x577000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e845489a0) put 0x558e845489a0 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e845489b8 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558e84548a10 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84548a80 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x579000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548a10) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548a98 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x579000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e84548a80) put 0x558e84548a80 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548a98 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558e84548af0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84548b60 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x57b000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548af0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548b78 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x57b000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e84548b60) put 0x558e84548b60 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548b78 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558e84548bd0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84548c40 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x57d000~2000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548bd0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548c58 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x57d000~2000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e84548c40) put 0x558e84548c40 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548c58 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x14000~1000: 0x0~1000 Blob(0x558e84548cb0 blob([!~1000] csum crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558e84548d20 sbid 0x0)) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x57f000~1000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548cb0) discard_unallocated 0x0~1000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548d38 in 0x558e80219960) _discard 0x0~1000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x57f000~1000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e84548d20) put 0x558e84548d20 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84548d38 in 0x558e80219960) _clear 2019-12-07T19:34:06.697+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.8s5_head 5#1:15219530:::smithi13140796-21:4#1a = 0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.8s5_head 5#1:15219530:::smithi13140796-21:head#1b onode 0x558e84315b80 txc 0x558e83d4cc00 2019-12-07T19:34:06.697+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.8s5_head 5#1:15219530:::smithi13140796-21:head#1b 0x0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.8s5_head 5#1:15219530:::smithi13140796-21:head#1b = 0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.697+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:06.697+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4cc00 cost 5729 (1 ios * 4000 + 1729 bytes) 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4cc00 onodes shared_blobs 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4cc00 allocated 0x[] released 0x[4ba000~20000,56b000~15000] 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv release 0x4ba000~20000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 10 freelist release 0x4ba000~20000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 freelist _xor first_key 0x480000 last_key 0x480000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv release 0x56b000~15000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 10 freelist release 0x56b000~15000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 freelist _xor first_key 0x500000 last_key 0x500000 2019-12-07T19:34:06.697+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 prepare 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4cc00 2019-12-07T19:34:06.697+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 io_done 2019-12-07T19:34:06.697+0000 7fe2c7a66700 10 osd.0 29 dequeue_op 0x558e8477adc0 finish 2019-12-07T19:34:06.697+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.697+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1387 < min_flush_size 524288 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417d400 had 1 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84315900 had 1 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84315b80 had 1 2019-12-07T19:34:06.697+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1410 < min_flush_size 524288 2019-12-07T19:34:06.697+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1410 < min_flush_size 524288 2019-12-07T19:34:06.697+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.697+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x10f2be~582 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.697+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x10f2be 0x10f2be~582 to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~e0000 x_off 0xdf2be 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x2be 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x840 and padding block with 0x7c0 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x10f840 2019-12-07T19:34:06.697+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.697+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.697+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000119002s (0.000000078s flush + 0.000118924s kv commit) 2019-12-07T19:34:06.697+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.697+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.697+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4cc00> 2019-12-07T19:34:06.697+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.697+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 kv_submitted 2019-12-07T19:34:06.697+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4cc00 2019-12-07T19:34:06.697+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 finishing 2019-12-07T19:34:06.697+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4cc00 onodes 2019-12-07T19:34:06.697+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4cc00 done 2019-12-07T19:34:06.697+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:06.697+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4cc00 [4ba000~20000,56b000~15000] 2019-12-07T19:34:06.697+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release 0x4ba000~20000 2019-12-07T19:34:06.697+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release 0x56b000~15000 2019-12-07T19:34:06.697+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.697+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.697+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 122 ==== pg_info2(1.fs1 1.fs1 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83ebd100 con 0x558e818d7000 2019-12-07T19:34:06.697+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.fs1 epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:06.697+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.697+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 123 ==== MOSDECSubOpWrite(1.fs1 29/28 ECSubWrite(tid=90, reqid=osd.3.0:89, at_version=0'0, trim_to=0'0, roll_forward_to=29'12)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e84042800 con 0x558e818d7000 2019-12-07T19:34:06.697+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e8477af20 prio 127 cost 0 latency 0.000014 epoch 29 MOSDECSubOpWrite(1.fs1 29/28 ECSubWrite(tid=90, reqid=osd.3.0:89, at_version=0'0, trim_to=0'0, roll_forward_to=29'12)) v2 2019-12-07T19:34:06.697+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 29/28 ECSubWrite(tid=90, reqid=osd.3.0:89, at_version=0'0, trim_to=0'0, roll_forward_to=29'12)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.697+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.697+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.697+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.697+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e29) pg 0x558e81a9b000 2019-12-07T19:34:06.697+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:06.697+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e83c56000 seq 33 2019-12-07T19:34:06.698+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56000 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56000 onodes shared_blobs 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56000 allocated 0x[] released 0x[] 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 prepare 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56000 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 io_done 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 29/28 ECSubWrite(tid=90, reqid=osd.3.0:89, at_version=0'0, trim_to=0'0, roll_forward_to=29'12)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGOpItem(op=MOSDECSubOpWrite(1.fs1 29/28 ECSubWrite(tid=90, reqid=osd.3.0:89, at_version=0'0, trim_to=0'0, roll_forward_to=29'12)) v2) prio 127 cost 0 e29) pg 0x558e81a9b000 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 osd.0 29 dequeue_op 0x558e8477af20 prio 127 cost 0 latency 0.000521 MOSDECSubOpWrite(1.fs1 29/28 ECSubWrite(tid=90, reqid=osd.3.0:89, at_version=0'0, trim_to=0'0, roll_forward_to=29'12)) v2 pg pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] _handle_message: MOSDECSubOpWrite(1.fs1 29/28 ECSubWrite(tid=90, reqid=osd.3.0:89, at_version=0'0, trim_to=0'0, roll_forward_to=29'12)) v2 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] append_log log((0'0,29'12], crt=27'10) [] 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 osd.0 pg_epoch: 29 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] rollforward: entry=29'11 (27'7) delete 1:f7919c75:::smithi13140796-20:4 by unknown.0.0:0 2019-12-07T19:34:06.691839+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 osd.0 pg_epoch: 29 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] rollforward: entry=29'12 (27'10) modify 1:f7919c75:::smithi13140796-20:head by unknown.0.0:0 2019-12-07T19:34:06.691839+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] append_log approx pg log length = 12 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.698+0000 7fe2c8267700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e83c56900 seq 34 2019-12-07T19:34:06.698+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4#b onode 0x558e83c72280 txc 0x558e83c56900 2019-12-07T19:34:06.698+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4#b 0x0 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore.blob(0x558e83f95650) put_ref 0x0~2000 Blob(0x558e83f95650 blob([0x9e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore.blob(0x558e811d0a10) put_ref 0x0~2000 Blob(0x558e811d0a10 blob([0xa0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore.blob(0x558e841a2070) put_ref 0x0~2000 Blob(0x558e841a2070 blob([0xa2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore.blob(0x558e83f955e0) put_ref 0x0~2000 Blob(0x558e83f955e0 blob([0xa4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore.blob(0x558e83f95570) put_ref 0x0~2000 Blob(0x558e83f95570 blob([0xa6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore.blob(0x558e83f95490) put_ref 0x0~2000 Blob(0x558e83f95490 blob([0xa8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore.blob(0x558e83f95420) put_ref 0x0~2000 Blob(0x558e83f95420 blob([0xaa000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore.blob(0x558e83f953b0) put_ref 0x0~2000 Blob(0x558e83f953b0 blob([0xac000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore.blob(0x558e83f95340) put_ref 0x0~2000 Blob(0x558e83f95340 blob([0xae000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore.blob(0x558e83f952d0) put_ref 0x0~2000 Blob(0x558e83f952d0 blob([0xb0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore.extentmap(0x558e83c72388) dirty_range mark inline shard dirty 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e83f95650 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x9e000~2000] 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e811d0a10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xa0000~2000] 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e841a2070 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xa2000~2000] 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e83f955e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xa4000~2000] 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e83f95570 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xa6000~2000] 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e83f95490 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xa8000~2000] 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e83f95420 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xaa000~2000] 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558e83f953b0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xac000~2000] 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558e83f95340 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xae000~2000] 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558e83f952d0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=2)))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xb0000~2000] 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove gen and maybe_unshared_blobs 0x558e8194e4d0,0x558e8194e8c0,0x558e8194f810,0x558e83af92d0,0x558e83af93b0,0x558e83af9490,0x558e83af9570,0x558e83c65ab0,0x558e83c65b20,0x558e83c65c00 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.fs1_head 1#1:f7919c75:::smithi13140796-20:4#b = 0 2019-12-07T19:34:06.698+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56900 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56900 onodes shared_blobs 0x558e8194e4d0,0x558e8194e8c0,0x558e8194f810,0x558e83af92d0,0x558e83af93b0,0x558e83af9490,0x558e83af9570,0x558e83c65ab0,0x558e83c65b20,0x558e83c65c00 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2804 is 11 SharedBlob(0x558e8194e4d0 loaded (sbid 0x2804 ref_map(0x9e000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2808 is 11 SharedBlob(0x558e8194e8c0 loaded (sbid 0x2808 ref_map(0xa2000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2806 is 11 SharedBlob(0x558e8194f810 loaded (sbid 0x2806 ref_map(0xa0000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x280e is 11 SharedBlob(0x558e83af92d0 loaded (sbid 0x280e ref_map(0xaa000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x280f is 11 SharedBlob(0x558e83af93b0 loaded (sbid 0x280f ref_map(0xac000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2810 is 11 SharedBlob(0x558e83af9490 loaded (sbid 0x2810 ref_map(0xae000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2811 is 11 SharedBlob(0x558e83af9570 loaded (sbid 0x2811 ref_map(0xb0000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x280c is 11 SharedBlob(0x558e83c65ab0 loaded (sbid 0x280c ref_map(0xa6000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x280a is 11 SharedBlob(0x558e83c65b20 loaded (sbid 0x280a ref_map(0xa4000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x280d is 11 SharedBlob(0x558e83c65c00 loaded (sbid 0x280d ref_map(0xa8000~2000=1))) 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56900 allocated 0x[] released 0x[] 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 prepare 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56900 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 io_done 2019-12-07T19:34:06.698+0000 7fe2c8267700 10 osd.0 29 dequeue_op 0x558e8477af20 finish 2019-12-07T19:34:06.698+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:06.698+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 223 ==== MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=147, reqid=osd.1.0:147, at_version=29'29, trim_to=0'0, roll_forward_to=29'27)) v2 ==== 2632+0+0 (unknown 0 0 0) 0x558e84041000 con 0x558e818d7c00 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 2 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1984 < min_flush_size 524288 2019-12-07T19:34:06.698+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e8477b080 prio 127 cost 0 latency 0.000031 epoch 29 MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=147, reqid=osd.1.0:147, at_version=29'29, trim_to=0'0, roll_forward_to=29'27)) v2 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2582 < min_flush_size 524288 2019-12-07T19:34:06.698+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=147, reqid=osd.1.0:147, at_version=29'29, trim_to=0'0, roll_forward_to=29'27)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72280 had 1 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2605 < min_flush_size 524288 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2605 < min_flush_size 524288 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x10f840~a2d to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x10f840 0x10f840~a2d to file(ino 16 size 0x110000 mtime 2019-12-07T19:34:06.681614+0000 allocated 110000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~e0000]) 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _allocate len 0x26d from 0 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _allocate len 0x26d from 1 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd240000 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd240000~10000/10000,10000,acd240000 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x120000 to include allocated 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 36 (was clean) 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xdf840 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x840 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x26d and padding block with 0xd93 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x120000 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=147, reqid=osd.1.0:147, at_version=29'29, trim_to=0'0, roll_forward_to=29'27)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=147, reqid=osd.1.0:147, at_version=29'29, trim_to=0'0, roll_forward_to=29'27)) v2) prio 127 cost 0 e29) pg 0x558e841a4000 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 osd.0 29 dequeue_op 0x558e8477b080 prio 127 cost 0 latency 0.000144 MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=147, reqid=osd.1.0:147, at_version=29'29, trim_to=0'0, roll_forward_to=29'27)) v2 pg pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'27 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'27 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=147, reqid=osd.1.0:147, at_version=29'29, trim_to=0'0, roll_forward_to=29'27)) v2 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 snap_mapper.remove_oid 1:131421e8:::smithi13140796-7:4 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 snap_mapper._remove_oid 1:131421e8:::smithi13140796-7:4 2019-12-07T19:34:06.698+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 snap_mapper.get_snaps 1:131421e8:::smithi13140796-7:4 4 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 snap_mapper.clear_snaps 1:131421e8:::smithi13140796-7:4 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 snap_mapper.clear_snaps rm OBJ_.5_0000000000000001.8C824871.4.smithi13140796-7.. 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 snap_mapper._remove_oid rm SNA_1_0000000000000004_.5_0000000000000001.8C824871.4.smithi13140796-7.. 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'27 (0'0,29'27] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'27 active mbc={} ps=[2~1]] append_log log((0'0,29'27], crt=29'27) [29'28 (27'20) delete 1:131421e8:::smithi13140796-7:4 by unknown.0.0:0 2019-12-07T19:34:06.698406+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,29'29 (27'21) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:06.698406+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'28 (0'0,29'28] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=29'27 crt=29'27 lcod 29'27 active mbc={} ps=[2~1]] add_log_entry 29'28 (27'20) delete 1:131421e8:::smithi13140796-7:4 by unknown.0.0:0 2019-12-07T19:34:06.698406+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=29'27 crt=29'27 lcod 29'27 active mbc={} ps=[2~1]] add_log_entry 29'29 (27'21) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:06.698406+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=29'27 crt=29'27 lcod 29'27 active mbc={} ps=[2~1]] append_log approx pg log length = 29 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=29'27 crt=29'27 lcod 29'27 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.698+0000 7fe2cba6e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 29'28, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e81a0b500 seq 30 2019-12-07T19:34:06.698+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:131421e8:::smithi13140796-7:4#1c key 0x858000000000000001131421e821736d'ithi13140796-7!='0x0000000000000004000000000000001c'o' 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11026d 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) r -2 v.len 0 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.698+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.8s5_head 5#1:131421e8:::smithi13140796-7:4# -> 5#1:131421e8:::smithi13140796-7:4#1c 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.8s5_head 5#1:131421e8:::smithi13140796-7:4# -> 5#1:131421e8:::smithi13140796-7:4#1c = 0 2019-12-07T19:34:06.698+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# 2 keys = 0 2019-12-07T19:34:06.698+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.8s5_head 5#1:131421e8:::smithi13140796-7:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:06.698+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.8C824871.4.smithi13140796-7..' <- OBJ_.5_0000000000000001.8C824871.4.smithi13140796-7.. 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.698+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_0000000000000004_.5_0000000000000001.8C824871.4.smithi13140796-7..' <- SNA_1_0000000000000004_.5_0000000000000001.8C824871.4.smithi13140796-7.. 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:06.698+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000029.00000000000000000028' <- 0000000029.00000000000000000028 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000029.00000000000000000029' <- 0000000029.00000000000000000029 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b500 cost 7506 (1 ios * 4000 + 3506 bytes) 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b500 onodes 0x558e811dc780,0x558e83b09400 shared_blobs 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e811dc888) update 5#1:131421e8:::smithi13140796-7:4#1c 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:131421e8:::smithi13140796-7:4#1c is 626 (411 bytes onode + 2 bytes spanning blobs + 213 bytes inline extents) 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e83b09508) update 5#1:131421e8:::smithi13140796-7:head# 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:131421e8:::smithi13140796-7:head# is 794 (484 bytes onode + 2 bytes spanning blobs + 308 bytes inline extents) 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b500 allocated 0x[] released 0x[] 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 prepare 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b500 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 io_done 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=29'27 crt=29'27 lcod 29'27 active mbc={} ps=[2~1]] op_applied version 29'29 2019-12-07T19:34:06.698+0000 7fe2cba6e700 10 osd.0 29 dequeue_op 0x558e8477b080 finish 2019-12-07T19:34:06.698+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 29/28 ECSubWriteReply(tid=146, last_complete=29'27, committed=1, applied=1)) v2 -- 0x558e81954000 con 0x558e818d7c00 2019-12-07T19:34:06.698+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (36) on file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]), flushing log 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:06.698+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 36 len 0x49 crc 0x1961792d) 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x23000~1000 to file(ino 1 size 0x23000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x23000 0x23000~1000 to file(ino 1 size 0x23000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x24000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x23000 2019-12-07T19:34:06.698+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x24000 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:06.698+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:34:06.699+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000190 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 36 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000769070s (0.000000060s flush + 0.000769010s kv commit) 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.699+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3841 < min_flush_size 524288 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dc780 had 1 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b09400 had 1 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2280 had 1 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:06.699+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3864 < min_flush_size 524288 2019-12-07T19:34:06.699+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3864 < min_flush_size 524288 2019-12-07T19:34:06.699+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.699+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11026d~f18 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.699+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11026d 0x11026d~f18 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe026d 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x26d 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x185 and padding block with 0xe7b 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56000,0x558e83c56900> 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.699+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 kv_submitted 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56000 2019-12-07T19:34:06.699+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56000 finishing 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56000 onodes 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56000 done 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56900 kv_submitted 2019-12-07T19:34:06.699+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56000 [] 2019-12-07T19:34:06.699+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.699+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 kv_submitted 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56900 2019-12-07T19:34:06.699+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x111185 2019-12-07T19:34:06.699+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.699+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.699+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000114288s (0.000000039s flush + 0.000114249s kv commit) 2019-12-07T19:34:06.699+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.699+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56900 finishing 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56900 onodes 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56900 done 2019-12-07T19:34:06.699+0000 7fe2c8267700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpWriteReply(1.fs0 29/28 ECSubWriteReply(tid=90, last_complete=29'12, committed=1, applied=1)) v2 -- 0x558e84029600 con 0x558e818d7000 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:06.699+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56900 [] 2019-12-07T19:34:06.699+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0b500> 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.699+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 kv_submitted 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b500 2019-12-07T19:34:06.699+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 finishing 2019-12-07T19:34:06.699+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b500 onodes 0x558e811dc780,0x558e83b09400 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b500 done 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:06.699+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b500 [] 2019-12-07T19:34:06.699+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.699+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:06.699+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.699+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.699+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 29/28 ECSubWriteReply(tid=147, last_complete=29'29, committed=1, applied=1)) v2 -- 0x558e81954200 con 0x558e818d7c00 2019-12-07T19:34:06.699+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.701+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 224 ==== MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=148, reqid=osd.1.0:147, at_version=0'0, trim_to=0'0, roll_forward_to=29'29)) v2 ==== 909+0+0 (unknown 0 0 0) 0x558e84043800 con 0x558e818d7c00 2019-12-07T19:34:06.701+0000 7fe2e7d3c700 15 osd.0 29 enqueue_op 0x558e8477b1e0 prio 127 cost 0 latency 0.000019 epoch 29 MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=148, reqid=osd.1.0:147, at_version=0'0, trim_to=0'0, roll_forward_to=29'29)) v2 2019-12-07T19:34:06.701+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=148, reqid=osd.1.0:147, at_version=0'0, trim_to=0'0, roll_forward_to=29'29)) v2) prio 127 cost 0 e29) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.701+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 225 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e29/29) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83ebd100 con 0x558e818d7c00 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=148, reqid=osd.1.0:147, at_version=0'0, trim_to=0'0, roll_forward_to=29'29)) v2) prio 127 cost 0 e29) queued 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=148, reqid=osd.1.0:147, at_version=0'0, trim_to=0'0, roll_forward_to=29'29)) v2) prio 127 cost 0 e29) pg 0x558e841a4000 2019-12-07T19:34:06.701+0000 7fe2e7d3c700 15 osd.0 29 enqueue_peering_evt 1.8s5 epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:06.701+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e29) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 10 osd.0 29 dequeue_op 0x558e8477b1e0 prio 127 cost 0 latency 0.000051 MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=148, reqid=osd.1.0:147, at_version=0'0, trim_to=0'0, roll_forward_to=29'29)) v2 pg pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'27 active mbc={} ps=[2~1]] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'27 active mbc={} ps=[2~1]] _handle_message: MOSDECSubOpWrite(1.8s5 29/28 ECSubWrite(tid=148, reqid=osd.1.0:147, at_version=0'0, trim_to=0'0, roll_forward_to=29'29)) v2 2019-12-07T19:34:06.701+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:06.701+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e29) queued 2019-12-07T19:34:06.701+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'27 active mbc={} ps=[2~1]] append_log log((0'0,29'29], crt=29'27) [] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 osd.0 pg_epoch: 29 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1]] rollforward: entry=29'28 (27'20) delete 1:131421e8:::smithi13140796-7:4 by unknown.0.0:0 2019-12-07T19:34:06.698406+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 osd.0 pg_epoch: 29 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1]] rollforward: entry=29'29 (27'21) modify 1:131421e8:::smithi13140796-7:head by unknown.0.0:0 2019-12-07T19:34:06.698406+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1]] append_log approx pg log length = 29 2019-12-07T19:34:06.701+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1]] append_log transaction_applied = 1 2019-12-07T19:34:06.701+0000 7fe2c7a66700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e819a0000 seq 31 2019-12-07T19:34:06.701+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.8s5_head 5#1:131421e8:::smithi13140796-7:4#1c onode 0x558e811dc780 txc 0x558e819a0000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.8s5_head 5#1:131421e8:::smithi13140796-7:4#1c 0x0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f61420) put_ref 0x0~2000 Blob(0x558e83f61420 blob([0x537000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f610a0 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5b3b0) put_ref 0x0~2000 Blob(0x558e83f5b3b0 blob([0x539000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5b490 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5b500) put_ref 0x0~2000 Blob(0x558e83f5b500 blob([0x53b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644cb0 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644d20) put_ref 0x0~2000 Blob(0x558e84644d20 blob([0x53d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644d90 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644e00) put_ref 0x0~2000 Blob(0x558e84644e00 blob([0x53f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644e70 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644ee0) put_ref 0x0~2000 Blob(0x558e84644ee0 blob([0x541000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644f50 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644fc0) put_ref 0x0~2000 Blob(0x558e84644fc0 blob([0x543000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645030 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e846450a0) put_ref 0x0~2000 Blob(0x558e846450a0 blob([0x545000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645110 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645180) put_ref 0x0~2000 Blob(0x558e84645180 blob([0x547000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846451f0 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645260) put_ref 0x0~2000 Blob(0x558e84645260 blob([0x549000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846452d0 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645340) put_ref 0x0~1000 Blob(0x558e84645340 blob([0x54b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e846453b0 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e811dc888) dirty_range mark inline shard dirty 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e83f61420 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f610a0 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x537000~2000] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f61420) discard_unallocated 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f610b8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x537000~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e83f610a0) put 0x558e83f610a0 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f610b8 in 0x558e80219960) _clear 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e83f5b3b0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83f5b490 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x539000~2000] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5b3b0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f5b4a8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x539000~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e83f5b490) put 0x558e83f5b490 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f5b4a8 in 0x558e80219960) _clear 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e83f5b500 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84644cb0 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x53b000~2000] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f5b500) discard_unallocated 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644cc8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x53b000~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e84644cb0) put 0x558e84644cb0 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644cc8 in 0x558e80219960) _clear 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e84644d20 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84644d90 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x53d000~2000] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644d20) discard_unallocated 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644da8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x53d000~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e84644d90) put 0x558e84644d90 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644da8 in 0x558e80219960) _clear 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e84644e00 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84644e70 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x53f000~2000] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644e00) discard_unallocated 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644e88 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x53f000~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e84644e70) put 0x558e84644e70 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644e88 in 0x558e80219960) _clear 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e84644ee0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84644f50 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x541000~2000] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644ee0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644f68 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x541000~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e84644f50) put 0x558e84644f50 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84644f68 in 0x558e80219960) _clear 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e84644fc0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84645030 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x543000~2000] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e84644fc0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645048 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x543000~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e84645030) put 0x558e84645030 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645048 in 0x558e80219960) _clear 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558e846450a0 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e84645110 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x545000~2000] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e846450a0) discard_unallocated 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645128 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x545000~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e84645110) put 0x558e84645110 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645128 in 0x558e80219960) _clear 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558e84645180 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e846451f0 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x547000~2000] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645180) discard_unallocated 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645208 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x547000~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e846451f0) put 0x558e846451f0 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645208 in 0x558e80219960) _clear 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558e84645260 blob([!~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e846452d0 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x549000~2000] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645260) discard_unallocated 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e846452e8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x549000~2000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e846452d0) put 0x558e846452d0 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e846452e8 in 0x558e80219960) _clear 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x14000~1000: 0x0~1000 Blob(0x558e84645340 blob([!~1000] csum crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558e846453b0 sbid 0x0)) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x54b000~1000] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645340) discard_unallocated 0x0~1000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e846453c8 in 0x558e80219960) _discard 0x0~1000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x54b000~1000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.sharedblob(0x558e846453b0) put 0x558e846453b0 removing self from set 0x558e8403edf0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e846453c8 in 0x558e80219960) _clear 2019-12-07T19:34:06.701+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.8s5_head 5#1:131421e8:::smithi13140796-7:4#1c = 0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:06.701+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:06.701+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0000 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0000 onodes shared_blobs 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0000 allocated 0x[] released 0x[537000~15000] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv release 0x537000~15000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 10 freelist release 0x537000~15000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 freelist _xor first_key 0x500000 last_key 0x500000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 prepare 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 io_done 2019-12-07T19:34:06.701+0000 7fe2c7a66700 10 osd.0 29 dequeue_op 0x558e8477b1e0 finish 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.701+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1249 < min_flush_size 524288 2019-12-07T19:34:06.701+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e29) pg 0x558e841a4000 2019-12-07T19:34:06.701+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1]] do_peering_event: epoch_sent: 29 epoch_requested: 29 MInfoRec from 1(0) info: 1.8s0( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:06.701+0000 7fe2cba6e700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1]] proc_primary_info updating purged_snaps to [2~1,4~1] 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dc780 had 1 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1272 < min_flush_size 524288 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1272 < min_flush_size 524288 2019-12-07T19:34:06.701+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x111185~4f8 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.701+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:06.701+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e81a0a900 seq 32 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x111185 0x111185~4f8 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.701+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe1185 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x185 2019-12-07T19:34:06.701+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._biginfo' <- _biginfo 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x67d and padding block with 0x983 2019-12-07T19:34:06.701+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:06.701+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:06.701+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a900 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:06.701+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a900 onodes shared_blobs 2019-12-07T19:34:06.701+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a900 allocated 0x[] released 0x[] 2019-12-07T19:34:06.701+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 prepare 2019-12-07T19:34:06.701+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a900 2019-12-07T19:34:06.701+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 io_done 2019-12-07T19:34:06.701+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11167d 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000209107s (0.000000082s flush + 0.000209025s kv commit) 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11167d~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11167d 0x11167d~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe167d 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x67d 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xaea and padding block with 0x516 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a0000> 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.701+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 kv_submitted 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0000 2019-12-07T19:34:06.701+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0000 finishing 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0000 onodes 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0000 done 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a900 kv_submitted 2019-12-07T19:34:06.701+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0000 [537000~15000] 2019-12-07T19:34:06.701+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release 0x537000~15000 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.701+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.701+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 29/28 ECSubWriteReply(tid=148, last_complete=29'29, committed=1, applied=1)) v2 -- 0x558e819df600 con 0x558e818d7c00 2019-12-07T19:34:06.701+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x111aea 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:06.701+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000231055s (0.000000068s flush + 0.000230987s kv commit) 2019-12-07T19:34:06.701+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0a900> 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:06.701+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 kv_submitted 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a900 2019-12-07T19:34:06.701+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a900 finishing 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a900 onodes 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a900 done 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:06.701+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a900 [] 2019-12-07T19:34:06.701+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.701+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:06.701+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:06.701+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:06.719+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.769+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.819+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.860+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.4 172.21.15.109:0/36938 3 ==== osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:06.860603+0000/30.309829762s send_stamp 30.309829762s delta_ub 18446744073.704274192s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e819df600 con 0x558e81985800 2019-12-07T19:34:06.860+0000 7fe2e7d3c700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703725999s,18446744073.704274192s]) 2019-12-07T19:34:06.860+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:06.860603+0000/30.309829762s send_stamp 30.315655379s delta_ub 0.005825617s) v5 -- 0x558e81954200 con 0x558e81985800 2019-12-07T19:34:06.860+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.4 172.21.15.109:0/36938 3 ==== osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:06.860603+0000/30.309829762s send_stamp 30.309829762s delta_ub 18446744073.704274192s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e81985400 2019-12-07T19:34:06.860+0000 7fe2e6d3a700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703687191s,18446744073.704274192s]) 2019-12-07T19:34:06.860+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:06.860603+0000/30.309829762s send_stamp 30.315694187s delta_ub 0.005864425s) v5 -- 0x558e81954e00 con 0x558e81985400 2019-12-07T19:34:06.869+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.919+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:06.969+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.020+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.053+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.1 172.21.15.131:0/37078 3 ==== osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.054100+0000/30.511023139s send_stamp 30.511023139s delta_ub 0.002321667s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81954000 con 0x558e81985000 2019-12-07T19:34:07.053+0000 7fe2e7d3c700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002138627s,0.002321667s]) 2019-12-07T19:34:07.053+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.054100+0000/30.511023139s send_stamp 30.508884512s delta_ub 18446744073.707412989s) v5 -- 0x558e81954200 con 0x558e81985000 2019-12-07T19:34:07.053+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.1 172.21.15.131:0/37078 3 ==== osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.054100+0000/30.511023139s send_stamp 30.511023139s delta_ub 0.002321667s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee800 con 0x558e81984c00 2019-12-07T19:34:07.053+0000 7fe2e6d3a700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002117893s,0.002321667s]) 2019-12-07T19:34:07.053+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.054100+0000/30.511023139s send_stamp 30.508905246s delta_ub 18446744073.707433723s) v5 -- 0x558e81954e00 con 0x558e81984c00 2019-12-07T19:34:07.070+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.120+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.170+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.220+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.271+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.321+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.333+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:07.371+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.421+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.443+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.5 172.21.15.109:0/36936 9 ==== osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.443495+0000/30.909818262s send_stamp 30.909818262s delta_ub 0.011725244s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e819df600 con 0x558e8182c800 2019-12-07T19:34:07.443+0000 7fe2e7d3c700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011277209s,0.011725244s]) 2019-12-07T19:34:07.443+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.443495+0000/30.909818262s send_stamp 30.898541053s delta_ub 18446744073.698274407s) v5 -- 0x558e81954200 con 0x558e8182c800 2019-12-07T19:34:07.443+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.5 172.21.15.109:0/36936 9 ==== osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.443495+0000/30.909818262s send_stamp 30.909818262s delta_ub 0.011725244s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e819df600 con 0x558e8181b800 2019-12-07T19:34:07.443+0000 7fe2e7d3c700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011189216s,0.011725244s]) 2019-12-07T19:34:07.443+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.443495+0000/30.909818262s send_stamp 30.898629046s delta_ub 18446744073.698362400s) v5 -- 0x558e84029600 con 0x558e8181b800 2019-12-07T19:34:07.471+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.518+0000 7fe2e2ca2700 10 osd.0 29 tick_without_osd_lock 2019-12-07T19:34:07.519+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fb20000/0x40000000/0x1680000000, data 0x4ae802/0x4de000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:07.519+0000 7fe2e2ca2700 25 osd.0 29 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254252+0000 last_rx_front 2019-12-07T19:34:02.254310+0000 2019-12-07T19:34:07.519+0000 7fe2e2ca2700 25 osd.0 29 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254273+0000 last_rx_front 2019-12-07T19:34:02.254232+0000 2019-12-07T19:34:07.519+0000 7fe2e2ca2700 25 osd.0 29 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254209+0000 last_rx_front 2019-12-07T19:34:02.254194+0000 2019-12-07T19:34:07.519+0000 7fe2e2ca2700 25 osd.0 29 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254409+0000 last_rx_front 2019-12-07T19:34:02.254290+0000 2019-12-07T19:34:07.519+0000 7fe2e2ca2700 25 osd.0 29 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254354+0000 last_rx_front 2019-12-07T19:34:02.254301+0000 2019-12-07T19:34:07.519+0000 7fe2e2ca2700 25 osd.0 29 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:02.253945+0000 last_rx_back 2019-12-07T19:34:02.254368+0000 last_rx_front 2019-12-07T19:34:02.254333+0000 2019-12-07T19:34:07.519+0000 7fe2e2ca2700 20 osd.0 29 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:07.519+0000 7fe2e2ca2700 10 osd.0 29 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:07.519+0000 7fe2e2ca2700 20 osd.0 29 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:07.519+0000 7fe2e2ca2700 10 osd.0 29 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:07.521+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.553+0000 7fe2c3a5e700 5 osd.0 29 heartbeat osd_stat(store_statfs(0x163fb20000/0x40000000/0x1680000000, data 0x4ae802/0x4de000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5,7] op hist []) 2019-12-07T19:34:07.553+0000 7fe2c3a5e700 20 osd.0 29 check_full_status cur ratio 5.28971e-05, physical ratio 5.28971e-05, new state none 2019-12-07T19:34:07.553+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.008970634s delta_ub 18446744073.707433723s) v5 -- 0x558e83ca8e00 con 0x558e818d7800 2019-12-07T19:34:07.553+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.008970634s delta_ub 18446744073.707433723s) v5 -- 0x558e83ca9000 con 0x558e818d7400 2019-12-07T19:34:07.553+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.008970634s delta_ub 18446744073.692714625s) v5 -- 0x558e83ca9200 con 0x558e817ff000 2019-12-07T19:34:07.553+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.008970634s delta_ub 18446744073.692714625s) v5 -- 0x558e83ca9400 con 0x558e817ff400 2019-12-07T19:34:07.553+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.008970634s delta_ub 18446744073.706893748s) v5 -- 0x558e83ca9600 con 0x558e817ff800 2019-12-07T19:34:07.553+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.008970634s delta_ub 18446744073.706893748s) v5 -- 0x558e83ca9800 con 0x558e817ffc00 2019-12-07T19:34:07.553+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.008970634s delta_ub 0.005864425s) v5 -- 0x558e83ca9a00 con 0x558e8181a000 2019-12-07T19:34:07.553+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.008970634s delta_ub 0.005864425s) v5 -- 0x558e83e68000 con 0x558e8181a400 2019-12-07T19:34:07.553+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.008970634s delta_ub 18446744073.698362400s) v5 -- 0x558e83e68200 con 0x558e817fe000 2019-12-07T19:34:07.553+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.008970634s delta_ub 18446744073.698362400s) v5 -- 0x558e83e68400 con 0x558e817fe400 2019-12-07T19:34:07.553+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.008970634s delta_ub 0.019741035s) v5 -- 0x558e83e68600 con 0x558e817fe800 2019-12-07T19:34:07.553+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.008970634s delta_ub 0.019741035s) v5 -- 0x558e83e68800 con 0x558e817fec00 2019-12-07T19:34:07.553+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6808/37077 8 ==== osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.026261300s delta_ub 0.017290666s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8400 con 0x558e817ff000 2019-12-07T19:34:07.553+0000 7fe2e753b700 25 osd.0 29 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:02.254273+0000 -> 2019-12-07T19:34:07.554614+0000 last_rx_front 2019-12-07T19:34:02.254232+0000 2019-12-07T19:34:07.553+0000 7fe2e753b700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016955099s,0.017290666s]) 2019-12-07T19:34:07.553+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6827/37078 4 ==== osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.011490449s delta_ub 0.002519815s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e818d7400 2019-12-07T19:34:07.553+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6831/37078 4 ==== osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.011432892s delta_ub 0.002462258s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8400 con 0x558e818d7800 2019-12-07T19:34:07.553+0000 7fe2e6d3a700 25 osd.0 29 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:02.254252+0000 last_rx_front 2019-12-07T19:34:02.254310+0000 -> 2019-12-07T19:34:07.554641+0000 2019-12-07T19:34:07.553+0000 7fe2e6d3a700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002156963s,0.002519815s]) 2019-12-07T19:34:07.553+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6816/37076 8 ==== osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.012112843s delta_ub 0.003142209s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81954000 con 0x558e817ff800 2019-12-07T19:34:07.553+0000 7fe2e7d3c700 25 osd.0 29 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:02.254209+0000 -> 2019-12-07T19:34:07.554661+0000 last_rx_front 2019-12-07T19:34:02.254194+0000 2019-12-07T19:34:07.553+0000 7fe2e7d3c700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002760184s,0.003142209s]) 2019-12-07T19:34:07.553+0000 7fe2e753b700 25 osd.0 29 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:02.254252+0000 -> 2019-12-07T19:34:07.554672+0000 last_rx_front 2019-12-07T19:34:07.554641+0000 2019-12-07T19:34:07.553+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6806/37077 8 ==== osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.026324663s delta_ub 0.017354029s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e817ff400 2019-12-07T19:34:07.553+0000 7fe2e753b700 25 osd.0 29 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:07.554278+0000) and older pending ping(s) 2019-12-07T19:34:07.554+0000 7fe2e753b700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002069114s,0.002462258s]) 2019-12-07T19:34:07.554+0000 7fe2e6d3a700 25 osd.0 29 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:07.554614+0000 last_rx_front 2019-12-07T19:34:02.254232+0000 -> 2019-12-07T19:34:07.554698+0000 2019-12-07T19:34:07.554+0000 7fe2e6d3a700 25 osd.0 29 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:07.554278+0000) and older pending ping(s) 2019-12-07T19:34:07.554+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6814/37076 8 ==== osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.012082733s delta_ub 0.003112099s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8400 con 0x558e817ffc00 2019-12-07T19:34:07.554+0000 7fe2e6d3a700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016934658s,0.017354029s]) 2019-12-07T19:34:07.554+0000 7fe2e753b700 25 osd.0 29 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:07.554661+0000 last_rx_front 2019-12-07T19:34:02.254194+0000 -> 2019-12-07T19:34:07.554715+0000 2019-12-07T19:34:07.554+0000 7fe2e753b700 25 osd.0 29 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:07.554278+0000) and older pending ping(s) 2019-12-07T19:34:07.554+0000 7fe2e753b700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002675183s,0.003112099s]) 2019-12-07T19:34:07.554+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6830/36939 8 ==== osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 30.990100066s delta_ub 18446744073.690681048s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e817fe800 2019-12-07T19:34:07.554+0000 7fe2e6d3a700 25 osd.0 29 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:02.254368+0000 -> 2019-12-07T19:34:07.554811+0000 last_rx_front 2019-12-07T19:34:02.254333+0000 2019-12-07T19:34:07.554+0000 7fe2e6d3a700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690148210s,18446744073.690681048s]) 2019-12-07T19:34:07.554+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6814/36938 8 ==== osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.003891854s delta_ub 18446744073.704472836s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e8181a000 2019-12-07T19:34:07.554+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6804/36936 8 ==== osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.021019863s delta_ub 0.012049229s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8400 con 0x558e817fe400 2019-12-07T19:34:07.554+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6806/36936 8 ==== osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.020997833s delta_ub 0.012027199s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81954000 con 0x558e817fe000 2019-12-07T19:34:07.554+0000 7fe2e6d3a700 25 osd.0 29 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:02.254409+0000 -> 2019-12-07T19:34:07.554842+0000 last_rx_front 2019-12-07T19:34:02.254290+0000 2019-12-07T19:34:07.554+0000 7fe2e6d3a700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703909631s,18446744073.704472836s]) 2019-12-07T19:34:07.554+0000 7fe2e753b700 25 osd.0 29 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:02.254354+0000 last_rx_front 2019-12-07T19:34:02.254301+0000 -> 2019-12-07T19:34:07.554854+0000 2019-12-07T19:34:07.554+0000 7fe2e753b700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011474196s,0.012049229s]) 2019-12-07T19:34:07.554+0000 7fe2e7d3c700 25 osd.0 29 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:02.254354+0000 -> 2019-12-07T19:34:07.554864+0000 last_rx_front 2019-12-07T19:34:07.554854+0000 2019-12-07T19:34:07.554+0000 7fe2e7d3c700 25 osd.0 29 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:07.554278+0000) and older pending ping(s) 2019-12-07T19:34:07.554+0000 7fe2e7d3c700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011442244s,0.012027199s]) 2019-12-07T19:34:07.554+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6828/36939 8 ==== osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 30.990139526s delta_ub 18446744073.690720508s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81954000 con 0x558e817fec00 2019-12-07T19:34:07.554+0000 7fe2e7d3c700 25 osd.0 29 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:07.554811+0000 last_rx_front 2019-12-07T19:34:02.254333+0000 -> 2019-12-07T19:34:07.554891+0000 2019-12-07T19:34:07.554+0000 7fe2e7d3c700 25 osd.0 29 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:34:07.554278+0000) and older pending ping(s) 2019-12-07T19:34:07.554+0000 7fe2e7d3c700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690108476s,18446744073.690720508s]) 2019-12-07T19:34:07.554+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6812/36938 8 ==== osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.554278+0000/31.008970634s send_stamp 31.003925530s delta_ub 18446744073.704506512s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e81954000 con 0x558e8181a400 2019-12-07T19:34:07.554+0000 7fe2e7d3c700 25 osd.0 29 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:07.554842+0000 last_rx_front 2019-12-07T19:34:02.254290+0000 -> 2019-12-07T19:34:07.554911+0000 2019-12-07T19:34:07.554+0000 7fe2e7d3c700 25 osd.0 29 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:07.554278+0000) and older pending ping(s) 2019-12-07T19:34:07.554+0000 7fe2e7d3c700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703874286s,18446744073.704506512s]) 2019-12-07T19:34:07.571+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.604+0000 7fe2e4535700 10 osd.0 29 tick 2019-12-07T19:34:07.604+0000 7fe2e4535700 10 osd.0 29 do_waiters -- start 2019-12-07T19:34:07.604+0000 7fe2e4535700 10 osd.0 29 do_waiters -- finish 2019-12-07T19:34:07.604+0000 7fe2e4535700 20 osd.0 29 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:07.621+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.671+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.721+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.736+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83ddac00 0x558e84058000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:07.736+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83ddac00 0x558e84058000 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:07.736+0000 7fe2e753b700 10 osd.0 29 ms_handle_authentication new session 0x558e83ff61a0 con 0x558e83ddac00 entity client.admin addr 2019-12-07T19:34:07.736+0000 7fe2e753b700 10 osd.0 29 ms_handle_authentication session 0x558e83ff61a0 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:07.736+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/3259582720 conn(0x558e83ddac00 0x558e84058000 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=client.4385 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:07.737+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4385 172.21.15.131:0/3259582720 1 ==== command(tid 1: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558e8122c1e0 con 0x558e83ddac00 2019-12-07T19:34:07.737+0000 7fe2d959f700 20 osd.0 29 OSD::ms_dispatch: command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:07.737+0000 7fe2d959f700 10 osd.0 29 do_waiters -- start 2019-12-07T19:34:07.737+0000 7fe2d959f700 10 osd.0 29 do_waiters -- finish 2019-12-07T19:34:07.737+0000 7fe2d959f700 20 osd.0 29 _dispatch 0x558e8122c1e0 command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:07.737+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/3259582720 -- command_reply(tid 1: 0 ) v1 -- 0x558e83c836c0 con 0x558e83ddac00 2019-12-07T19:34:07.757+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4385 172.21.15.131:0/3259582720 2 ==== command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558e8122cf00 con 0x558e83ddac00 2019-12-07T19:34:07.757+0000 7fe2d959f700 20 osd.0 29 OSD::ms_dispatch: command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:07.757+0000 7fe2d959f700 10 osd.0 29 do_waiters -- start 2019-12-07T19:34:07.757+0000 7fe2d959f700 10 osd.0 29 do_waiters -- finish 2019-12-07T19:34:07.757+0000 7fe2d959f700 20 osd.0 29 _dispatch 0x558e8122cf00 command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:07.758+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/3259582720 -- command_reply(tid 2: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558e83c83500 con 0x558e83ddac00 2019-12-07T19:34:07.771+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.822+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.872+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.912+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/3259582720 conn(0x558e83ddac00 msgr2=0x558e84058000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 61 2019-12-07T19:34:07.912+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/3259582720 conn(0x558e83ddac00 msgr2=0x558e84058000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:07.912+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/3259582720 conn(0x558e83ddac00 0x558e84058000 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:07.912+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/3259582720 conn(0x558e83ddac00 0x558e84058000 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:07.912+0000 7fe2d959f700 2 osd.0 29 ms_handle_reset con 0x558e83ddac00 session 0x558e83ff61a0 2019-12-07T19:34:07.912+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] reap_dead start 2019-12-07T19:34:07.922+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 505742 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:07.943+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.5 172.21.15.109:0/36936 10 ==== osd_ping(ping e30 up_from 13 ping_stamp 2019-12-07T19:34:07.943773+0000/31.410096593s send_stamp 31.410096593s delta_ub 0.012049229s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e819df600 con 0x558e8182c800 2019-12-07T19:34:07.943+0000 7fe2e7d3c700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011352814s,0.012049229s]) 2019-12-07T19:34:07.943+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.943773+0000/31.410096593s send_stamp 31.398743779s delta_ub 18446744073.698198802s) v5 -- 0x558e83e68800 con 0x558e8182c800 2019-12-07T19:34:07.943+0000 7fe2e7d3c700 10 osd.0 29 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 29 -> 30 (as per caller) 2019-12-07T19:34:07.943+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.5 172.21.15.109:0/36936 10 ==== osd_ping(ping e30 up_from 13 ping_stamp 2019-12-07T19:34:07.943773+0000/31.410096593s send_stamp 31.410096593s delta_ub 0.012049229s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e819df600 con 0x558e8181b800 2019-12-07T19:34:07.943+0000 7fe2e7d3c700 20 osd.0 29 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011314380s,0.012049229s]) 2019-12-07T19:34:07.943+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e29 up_from 13 ping_stamp 2019-12-07T19:34:07.943773+0000/31.410096593s send_stamp 31.398782213s delta_ub 18446744073.698237236s) v5 -- 0x558e83e68200 con 0x558e8181b800 2019-12-07T19:34:07.943+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 58 ==== osd_map(30..30 src has 1..30) v4 ==== 633+0+0 (unknown 0 0 0) 0x558e846e2480 con 0x558e818d6800 2019-12-07T19:34:07.943+0000 7fe2d4a80700 20 osd.0 29 OSD::ms_dispatch: osd_map(30..30 src has 1..30) v4 2019-12-07T19:34:07.943+0000 7fe2d4a80700 10 osd.0 29 do_waiters -- start 2019-12-07T19:34:07.943+0000 7fe2d4a80700 10 osd.0 29 do_waiters -- finish 2019-12-07T19:34:07.943+0000 7fe2d4a80700 20 osd.0 29 _dispatch 0x558e846e2480 osd_map(30..30 src has 1..30) v4 2019-12-07T19:34:07.943+0000 7fe2d4a80700 3 osd.0 29 handle_osd_map epochs [30,30], i have 29, src has [1,30] 2019-12-07T19:34:07.943+0000 7fe2d4a80700 10 osd.0 29 handle_osd_map got inc map for epoch 30 2019-12-07T19:34:07.943+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3bd8e826:::osdmap.29:0# 0x0~0 2019-12-07T19:34:07.943+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1438 size 0x1438 (5176) 2019-12-07T19:34:07.943+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:07.943+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a7ba40 blob([0x584000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e81a7b9d0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:07.943+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423c770 blob([!~1000,0x585000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558e8423c7e0 sbid 0x0)) need 0x1000~438 cache has 0x[1000~438] 2019-12-07T19:34:07.943+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3bd8e826:::osdmap.29:0# 0x0~1438 = 5176 2019-12-07T19:34:07.943+0000 7fe2d4a80700 10 osd.0 29 add_map_bl 29 5176 bytes 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 osd.0 29 got_full_map 30, nothing requested 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 osd.0 29 handle_osd_map pg_num_history pg_num_history(e30 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps purged_snaps {30={}} 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 osd.0 29 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e30 [1,30] lci=[13,30]) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e83e72300 seq 22 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c4cb972f:::inc_osdmap.30:0# key 0x7f7fffffffffffffffc4cb97'/!inc_osdmap.30!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c4cb972f:::inc_osdmap.30:0# 0x0~24d 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1149 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c4cb972f:::inc_osdmap.30:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24d 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72300 1 blobs 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x37000~1000/1000,1000,0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x37000~1000] 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84190150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846c4380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84190150 blob([0x37000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c4380 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.blob(0x558e84190150) get_ref 0x0~24d Blob(0x558e84190150 blob([0x37000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c4380 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.blob(0x558e84190150) get_ref init 0x1000, 1000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558e84190150 blob([0x37000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558e846c4380 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e846c4398 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x24d 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e811dcd88) dirty_range mark inline shard dirty 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c4cb972f:::inc_osdmap.30:0# 0x0~24d = 0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3c18e826:::osdmap.30:0# key 0x7f7fffffffffffffff3c18e8'&!osdmap.30!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3c18e826:::osdmap.30:0# 0x0~1438 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1150 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3c18e826:::osdmap.30:0# 0x0~1438 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~438 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbc8 on front/back, now 0x0~1000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72300 2 blobs 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x38000~2000/1000,2000,0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x38000~2000] 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841902a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e845499d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841902a0 blob([0x38000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e845499d0 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.blob(0x558e841902a0) get_ref 0x0~1000 Blob(0x558e841902a0 blob([0x38000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e845499d0 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.blob(0x558e841902a0) get_ref init 0x1000, 1000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e841902a0 blob([0x38000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e845499d0 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e845499e8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841908c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6d490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841908c0 blob([!~1000,0x39000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6d490 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.blob(0x558e841908c0) get_ref 0x1000~438 Blob(0x558e841908c0 blob([!~1000,0x39000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e83e6d490 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.blob(0x558e841908c0) get_ref init 0x2000, 1000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~438: 0x1000~438 Blob(0x558e841908c0 blob([!~1000,0x39000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558e83e6d490 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e83e6d4a8 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1438 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e819b6388) dirty_range mark inline shard dirty 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3c18e826:::osdmap.30:0# 0x0~1438 = 0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72300 0 blobs 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72300 0 blobs 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72300 cost 23246 (4 ios * 4000 + 7246 bytes) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72300 onodes 0x558e80251b80,0x558e811dcc80,0x558e817b9180,0x558e819b6280 shared_blobs 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e811dcd88) update #-1:c4cb972f:::inc_osdmap.30:0# 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e811dcd88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c4cb972f:::inc_osdmap.30:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e819b6388) update #-1:3c18e826:::osdmap.30:0# 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e819b6388) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3c18e826:::osdmap.30:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72300 allocated 0x[37000~3000] released 0x[] 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 freelist allocate 0x37000~3000 2019-12-07T19:34:07.944+0000 7fe2d4a80700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72300 prepare 2019-12-07T19:34:07.944+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83e72300 2019-12-07T19:34:07.944+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72300 aio_wait 2019-12-07T19:34:07.944+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72300 2019-12-07T19:34:07.944+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72300 io_done 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:07.944+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dcc80 had 1 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6280 had 1 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:07.944+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3859677184 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:07.944+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998491) bluestore 86 GiB free (0.959947), bluefs_ratio 0.0399446 2019-12-07T19:34:07.944+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:07.944+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:07.944+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:07.944+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x111aea~226c to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:07.944+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x111aea 0x111aea~226c to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe1aea 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xaea 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd56 and padding block with 0x2aa 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x113d56 2019-12-07T19:34:07.944+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:07.944+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:07.944+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000145689s (0.000004821s flush + 0.000140868s kv commit) 2019-12-07T19:34:07.944+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:07.944+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:07.944+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e72300> 2019-12-07T19:34:07.944+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:07.944+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72300 kv_submitted 2019-12-07T19:34:07.944+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72300 2019-12-07T19:34:07.944+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e83e72300 osr 0x558e8107a500 2019-12-07T19:34:07.944+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e846eb040) _discard drop 19 0x1b000~1000 2019-12-07T19:34:07.944+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e846eb040) prepare_write seq 20 0x1b000~1000 crc f8683679 2019-12-07T19:34:07.944+0000 7fe2d7285700 10 osd.0 29 _committed_osd_maps 30..30 2019-12-07T19:34:07.944+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e846eb040) _discard drop 19 0x2000~1000 2019-12-07T19:34:07.944+0000 7fe2d7285700 10 osd.0 29 advance to epoch 30 (<= last 30 <= newest_map 30) 2019-12-07T19:34:07.944+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e846eb040) prepare_write seq 20 0x2000~1000 crc 8b586098 2019-12-07T19:34:07.944+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:07.944+0000 7fe2d7285700 7 osd.0 30 consume_map version 30 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 30 identify_splits_and_merges 1.8s5 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 30 identify_splits_and_merges 1.9s3 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 30 identify_splits_and_merges 1.1s1 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 30 identify_splits_and_merges 1.as3 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 30 identify_splits_and_merges 1.2s0 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 30 identify_splits_and_merges 1.3s5 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 30 identify_splits_and_merges 1.bs1 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:3.identify_splits_and_merges identify_splits_and_merges slot 1.bs3 has no pg and waiting_for_split 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 30 identify_splits_and_merges 1.4s1 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 30 identify_splits_and_merges 1.5s4 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 30 identify_splits_and_merges 1.ds5 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 30 identify_splits_and_merges 1.es4 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 30 identify_splits_and_merges 1.7s3 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 30 identify_splits_and_merges 1.fs1 e29 to e30 pg_nums {18=16} 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 30 prune_pg_created 2019-12-07T19:34:07.944+0000 7fe2d7285700 10 osd.0:0.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 2019-12-07T19:34:07.944+0000 7fe2d7285700 10 osd.0:1.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:07.944+0000 7fe2d7285700 10 osd.0:2.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:07.944+0000 7fe2d7285700 10 osd.0:3.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs3 empty, pruning 2019-12-07T19:34:07.944+0000 7fe2d7285700 10 osd.0:4.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:07.944+0000 7fe2d7285700 10 osd.0:5.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:07.944+0000 7fe2d7285700 10 osd.0:6.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:07.944+0000 7fe2d7285700 10 osd.0:7.consume_map 30 (was 29) 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:07.944+0000 7fe2d7285700 10 osd.0 30 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:07.944+0000 7fe2d7285700 15 osd.0 30 enqueue_peering_evt 1.8s5 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7fe2d7285700 15 osd.0 30 enqueue_peering_evt 1.9s3 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7fe2d7285700 15 osd.0 30 enqueue_peering_evt 1.1s1 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7fe2d7285700 15 osd.0 30 enqueue_peering_evt 1.as3 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7fe2d7285700 15 osd.0 30 enqueue_peering_evt 1.2s0 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7fe2d7285700 15 osd.0 30 enqueue_peering_evt 1.3s5 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7fe2d7285700 15 osd.0 30 enqueue_peering_evt 1.bs1 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7fe2d7285700 15 osd.0 30 enqueue_peering_evt 1.4s1 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7fe2d7285700 15 osd.0 30 enqueue_peering_evt 1.5s4 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7fe2d7285700 15 osd.0 30 enqueue_peering_evt 1.ds5 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7fe2d7285700 15 osd.0 30 enqueue_peering_evt 1.es4 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7fe2d7285700 15 osd.0 30 enqueue_peering_evt 1.7s3 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7fe2d7285700 15 osd.0 30 enqueue_peering_evt 1.fs1 epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.944+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) 2019-12-07T19:34:07.944+0000 7fe2d7285700 7 osd.0 30 activate_map version 30 2019-12-07T19:34:07.944+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.944+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.944+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558e81938000 2019-12-07T19:34:07.944+0000 7fe2cc26f700 20 osd.0 30 agent_entry empty queue 2019-12-07T19:34:07.944+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 29 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 20 osd.0:4.update_pg_epoch 1.4s1 29 -> 30 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 29 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558e83d07000 2019-12-07T19:34:07.945+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 30 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.945+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.945+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558e81a98000 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 30 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.945+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.945+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:34:07.945+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558e841a4000 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.5s4 29 -> 30 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 30 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 30 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 29 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] handle_advance_map [2,1,3,4,0,5]/[2,1,3,4,0,5] -- 2/2 2019-12-07T19:34:07.945+0000 7fe2c4a60700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 30 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.945+0000 7fe2c4a60700 20 osd.0:6.update_pg_epoch 1.es4 29 -> 30 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 20 osd.0 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.945+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 30 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.945+0000 7fe2c7a66700 20 osd.0:0.update_pg_epoch 1.8s5 29 -> 30 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 30 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.945+0000 7fe2c4a60700 10 osd.0 pg_epoch: 29 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,3,4,5,0,2]/[1,3,4,5,0,2] -- 1/1 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 30 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.945+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.5s4_head options 2019-12-07T19:34:07.945+0000 7fe2c7a66700 10 osd.0 pg_epoch: 29 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,4,5,2,3,0]/[1,4,5,2,3,0] -- 1/1 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 30 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 30 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.945+0000 7fe2c4a60700 10 osd.0 pg_epoch: 30 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 30 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 30 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 pg_epoch: 30 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.945+0000 7fe2c7a66700 10 osd.0 pg_epoch: 30 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.945+0000 7fe2c4a60700 10 osd.0 pg_epoch: 30 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 pg_epoch: 30 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 30 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:34:07.945+0000 7fe2c7a66700 10 osd.0 pg_epoch: 30 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.945+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:07.945+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.8s5_head options 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 30 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.945+0000 7fe2c4a60700 10 osd.0 pg_epoch: 30 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.945+0000 7fe2c7a66700 10 osd.0 pg_epoch: 30 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 30 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c4a60700 20 osd.0 pg_epoch: 30 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c7a66700 20 osd.0 pg_epoch: 30 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558e81231000 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 pg_epoch: 30 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.945+0000 7fe2c4a60700 20 osd.0 pg_epoch: 30 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c4a60700 20 osd.0 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.945+0000 7fe2c7a66700 20 osd.0 pg_epoch: 30 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.945+0000 7fe2c7a66700 20 osd.0 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 30 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c4a60700 10 osd.0 pg_epoch: 30 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.945+0000 7fe2c7a66700 10 osd.0 pg_epoch: 30 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c4a60700 10 osd.0 pg_epoch: 30 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558e819e7000 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0:3.update_pg_epoch 1.3s5 29 -> 30 2019-12-07T19:34:07.945+0000 7fe2c7a66700 10 osd.0 pg_epoch: 30 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.945+0000 7fe2c4a60700 20 osd.0 pg_epoch: 30 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.945+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:07.945+0000 7fe2c7a66700 20 osd.0 pg_epoch: 30 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.945+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 29 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.945+0000 7fe2c4a60700 10 osd.0 pg_epoch: 30 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.ds5 29 -> 30 2019-12-07T19:34:07.945+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 30 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.945+0000 7fe2c7a66700 10 osd.0 pg_epoch: 30 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.945+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:07.945+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 29 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 30 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558e8122e000 2019-12-07T19:34:07.945+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.3s5_head options 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 30 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 30 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 30 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.945+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558e8193b000 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 pg_epoch: 30 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 30 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558e8192b000 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 pg_epoch: 30 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 pg_epoch: 30 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 pg_epoch: 30 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.7s3 29 -> 30 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 30 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 30 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 30 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.as3 29 -> 30 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 30 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.9s3 29 -> 30 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 pg_epoch: 30 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 pg_epoch: 30 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 30 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,4,2,0,5,3]/[1,4,2,0,5,3] -- 1/1 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 30 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 30 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 osd.0 pg_epoch: 30 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.945+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.945+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 30 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 30 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558e83c78000 2019-12-07T19:34:07.945+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.945+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 30 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 30 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.945+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 pg_epoch: 30 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 pg_epoch: 30 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 pg_epoch: 30 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 30 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 pg_epoch: 30 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0:3.update_pg_epoch 1.bs1 29 -> 30 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 pg_epoch: 30 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 pg_epoch: 30 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 pg_epoch: 30 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 29 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 30 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 30 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 30 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 30 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 pg_epoch: 30 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 30 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 30 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs1_head options 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558e81075000 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 pg_epoch: 30 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 30 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 30 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 30 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 pg_epoch: 30 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) queued 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 pg_epoch: 30 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558e819e4000 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.2s0 29 -> 30 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 30 epoch_requested: 30 NullEvt) prio 255 cost 10 e30) pg 0x558e81a9b000 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 30 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 29 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.1s1 29 -> 30 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 30 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] state: Active advmap 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 29 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] handle_advance_map [3,0,2,4,5,1]/[3,0,2,4,5,1] -- 3/3 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 pg_epoch: 30 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] handle_advance_map: 30 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 30 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.fs1 29 -> 30 2019-12-07T19:34:07.945+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 30 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 osd.0 pg_epoch: 30 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.945+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:34:07.945+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 29 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] handle_advance_map [3,0,5,4,1,2]/[3,0,5,4,1,2] -- 3/3 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 30 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 pg_epoch: 30 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 pg_epoch: 30 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 30 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 30 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] state: Active: handling ActMap 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 30 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] Active: kicking snap trim 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 30 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 pg_epoch: 30 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:34:07.945+0000 7fe2c6a64700 7 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] state: handle ActMap primary 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 osd.0 pg_epoch: 30 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.945+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] prepare_stats_for_publish reporting purged_snaps [2~1,4~1] 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 30 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:07.945+0000 7fe2c6a64700 15 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] publish_stats_to_osd 29:88 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 pg_epoch: 30 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.945+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 pg_epoch: 30 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:07.945+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 30 advance_pg new pool opts old pool opts 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 30 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] handle_activate_map: 30 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 30 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 pg_epoch: 30 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 30 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 osd.0 pg_epoch: 30 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 30 epoch_requested: 30 NullEvt 2019-12-07T19:34:07.945+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:07.945+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:07.972+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 508978 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.022+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 508978 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.072+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 508978 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.104+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:08.122+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 508978 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.173+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 508978 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.223+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 508978 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.273+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 508978 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.306+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:08.323+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 508978 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.373+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 508978 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.423+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 508978 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.473+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 508978 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.481+0000 7fe2e2ca2700 10 osd.0 30 tick_without_osd_lock 2019-12-07T19:34:08.481+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fb1d000/0x40000000/0x1680000000, data 0x4afe87/0x4e1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:08.481+0000 7fe2e2ca2700 25 osd.0 30 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:07.554672+0000 last_rx_front 2019-12-07T19:34:07.554641+0000 2019-12-07T19:34:08.481+0000 7fe2e2ca2700 25 osd.0 30 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:07.554614+0000 last_rx_front 2019-12-07T19:34:07.554698+0000 2019-12-07T19:34:08.481+0000 7fe2e2ca2700 25 osd.0 30 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:07.554661+0000 last_rx_front 2019-12-07T19:34:07.554715+0000 2019-12-07T19:34:08.481+0000 7fe2e2ca2700 25 osd.0 30 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:07.554842+0000 last_rx_front 2019-12-07T19:34:07.554911+0000 2019-12-07T19:34:08.481+0000 7fe2e2ca2700 25 osd.0 30 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:07.554864+0000 last_rx_front 2019-12-07T19:34:07.554854+0000 2019-12-07T19:34:08.481+0000 7fe2e2ca2700 25 osd.0 30 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:07.554278+0000 last_rx_back 2019-12-07T19:34:07.554811+0000 last_rx_front 2019-12-07T19:34:07.554891+0000 2019-12-07T19:34:08.481+0000 7fe2e2ca2700 20 osd.0 30 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:08.481+0000 7fe2e2ca2700 10 osd.0 30 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:08.481+0000 7fe2e2ca2700 20 osd.0 30 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:08.481+0000 7fe2e2ca2700 10 osd.0 30 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:08.523+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 508978 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.573+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 508978 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.600+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 27 ==== osd_map(30..31 src has 1..31) v4 ==== 1286+0+0 (crc 0 0 0) 0x558e846e3d40 con 0x558e80f39000 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 osd.0 30 OSD::ms_dispatch: osd_map(30..31 src has 1..31) v4 2019-12-07T19:34:08.600+0000 7fe2d959f700 10 osd.0 30 do_waiters -- start 2019-12-07T19:34:08.600+0000 7fe2d959f700 10 osd.0 30 do_waiters -- finish 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 osd.0 30 _dispatch 0x558e846e3d40 osd_map(30..31 src has 1..31) v4 2019-12-07T19:34:08.600+0000 7fe2d959f700 3 osd.0 30 handle_osd_map epochs [30,31], i have 30, src has [1,31] 2019-12-07T19:34:08.600+0000 7fe2d959f700 10 osd.0 30 handle_osd_map got inc map for epoch 31 2019-12-07T19:34:08.600+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3c18e826:::osdmap.30:0# 0x0~0 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1438 size 0x1438 (5176) 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841902a0 blob([0x38000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e845499d0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841908c0 blob([!~1000,0x39000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558e83e6d490 sbid 0x0)) need 0x1000~438 cache has 0x[1000~438] 2019-12-07T19:34:08.600+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3c18e826:::osdmap.30:0# 0x0~1438 = 5176 2019-12-07T19:34:08.600+0000 7fe2d959f700 10 osd.0 30 add_map_bl 30 5176 bytes 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 osd.0 30 got_full_map 31, nothing requested 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 osd.0 30 handle_osd_map pg_num_history pg_num_history(e31 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:08.600+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps purged_snaps {31={1=[2~1]}} 2019-12-07T19:34:08.600+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 bluestore.onode(0x558e8125c500).flush flush done 2019-12-07T19:34:08.600+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 1 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 bluestore.OmapIteratorImpl(0x558e8405e300) lower_bound to PSN__1_0000000000000001 key 0x00000000000000000000000000000403'.PSN__1_0000000000000001' 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 bluestore.OmapIteratorImpl(0x558e8405e300) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 snap_mapper._lookup_purged_snap pool 1 snap 1 key 'PSN__1_0000000000000001' lower_bound not found 2019-12-07T19:34:08.600+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 bluestore.onode(0x558e8125c500).flush flush done 2019-12-07T19:34:08.600+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 1 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 bluestore.OmapIteratorImpl(0x558e8405e300) lower_bound to PSN__1_0000000000000003 key 0x00000000000000000000000000000403'.PSN__1_0000000000000003' 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 bluestore.OmapIteratorImpl(0x558e8405e300) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:08.600+0000 7fe2d959f700 20 snap_mapper._lookup_purged_snap pool 1 snap 3 key 'PSN__1_0000000000000003' lower_bound not found 2019-12-07T19:34:08.600+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps rm 0 keys, set 1 keys 2019-12-07T19:34:08.600+0000 7fe2d959f700 10 osd.0 30 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e31 [1,31] lci=[13,31]) 2019-12-07T19:34:08.600+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e811fa900 seq 23 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:cf0b972f:::inc_osdmap.31:0# key 0x7f7fffffffffffffffcf0b97'/!inc_osdmap.31!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:08.601+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:cf0b972f:::inc_osdmap.31:0# 0x0~285 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1151 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:cf0b972f:::inc_osdmap.31:0# 0x0~285 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~285 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xd7b on front/back, now 0x0~1000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fa900 1 blobs 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x79000~1000/1000,1000,0 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x79000~1000] 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841a2d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84190770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841a2d90 blob([0x79000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84190770 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.blob(0x558e841a2d90) get_ref 0x0~285 Blob(0x558e841a2d90 blob([0x79000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84190770 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.blob(0x558e841a2d90) get_ref init 0x1000, 1000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~285: 0x0~285 Blob(0x558e841a2d90 blob([0x79000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x285) SharedBlob(0x558e84190770 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e84190788 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x285 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f29288) dirty_range mark inline shard dirty 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:cf0b972f:::inc_osdmap.31:0# 0x0~285 = 0 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3398e826:::osdmap.31:0# key 0x7f7fffffffffffffff3398e8'&!osdmap.31!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:08.601+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3398e826:::osdmap.31:0# 0x0~1470 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1152 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3398e826:::osdmap.31:0# 0x0~1470 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~470 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xb90 on front/back, now 0x0~1000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fa900 2 blobs 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x7a000~2000/1000,2000,0 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x7a000~2000] 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841903f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f95ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841903f0 blob([0x7a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f95ab0 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.blob(0x558e841903f0) get_ref 0x0~1000 Blob(0x558e841903f0 blob([0x7a000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f95ab0 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.blob(0x558e841903f0) get_ref init 0x1000, 1000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e841903f0 blob([0x7a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f95ab0 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e83f95ac8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f95b20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84549ab0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f95b20 blob([!~1000,0x7b000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84549ab0 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.blob(0x558e83f95b20) get_ref 0x1000~470 Blob(0x558e83f95b20 blob([!~1000,0x7b000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e84549ab0 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.blob(0x558e83f95b20) get_ref init 0x2000, 1000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~470: 0x1000~470 Blob(0x558e83f95b20 blob([!~1000,0x7b000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,470]) SharedBlob(0x558e84549ab0 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e84549ac8 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1470 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f29508) dirty_range mark inline shard dirty 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3398e826:::osdmap.31:0# 0x0~1470 = 0 2019-12-07T19:34:08.601+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fa900 0 blobs 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:08.601+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:08.601+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_0000000000000002' <- PSN__1_0000000000000002 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:08.601+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fa900 0 blobs 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa900 cost 23413 (4 ios * 4000 + 7413 bytes) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa900 onodes 0x558e80251b80,0x558e817b9180,0x558e83f29180,0x558e83f29400 shared_blobs 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f29288) update #-1:cf0b972f:::inc_osdmap.31:0# 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f29288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:cf0b972f:::inc_osdmap.31:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f29508) update #-1:3398e826:::osdmap.31:0# 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f29508) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3398e826:::osdmap.31:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa900 allocated 0x[79000~3000] released 0x[] 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 freelist allocate 0x79000~3000 2019-12-07T19:34:08.601+0000 7fe2d959f700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 prepare 2019-12-07T19:34:08.601+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e811fa900 2019-12-07T19:34:08.601+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 aio_wait 2019-12-07T19:34:08.601+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa900 2019-12-07T19:34:08.601+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 io_done 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:08.601+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8858 < min_flush_size 524288 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83f29180 had 1 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83f29400 had 1 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:08.601+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8881 < min_flush_size 524288 2019-12-07T19:34:08.601+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8881 < min_flush_size 524288 2019-12-07T19:34:08.601+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.601+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x113d56~22b1 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.601+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x113d56 0x113d56~22b1 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe3d56 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd56 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x7 and padding block with 0xff9 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x116007 2019-12-07T19:34:08.601+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.601+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.601+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000143061s (0.000003482s flush + 0.000139579s kv commit) 2019-12-07T19:34:08.601+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.601+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.601+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa900> 2019-12-07T19:34:08.601+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.601+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 kv_submitted 2019-12-07T19:34:08.601+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa900 2019-12-07T19:34:08.601+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e811fa900 osr 0x558e8107a500 2019-12-07T19:34:08.601+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e846eb040) _discard drop 20 0x1b000~1000 2019-12-07T19:34:08.601+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e846eb040) prepare_write seq 21 0x1b000~1000 crc e2b79a84 2019-12-07T19:34:08.601+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e846eb040) _discard drop 20 0x2000~1000 2019-12-07T19:34:08.601+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e846eb040) prepare_write seq 21 0x2000~1000 crc 306374a5 2019-12-07T19:34:08.601+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.601+0000 7fe2d7285700 10 osd.0 30 _committed_osd_maps 31..31 2019-12-07T19:34:08.601+0000 7fe2d7285700 10 osd.0 30 advance to epoch 31 (<= last 31 <= newest_map 31) 2019-12-07T19:34:08.601+0000 7fe2d7285700 7 osd.0 31 consume_map version 31 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 31 identify_splits_and_merges 1.8s5 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 31 identify_splits_and_merges 1.9s3 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 31 identify_splits_and_merges 1.1s1 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 31 identify_splits_and_merges 1.as3 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 31 identify_splits_and_merges 1.2s0 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 31 identify_splits_and_merges 1.3s5 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 31 identify_splits_and_merges 1.bs1 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 31 identify_splits_and_merges 1.4s1 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 31 identify_splits_and_merges 1.5s4 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 31 identify_splits_and_merges 1.ds5 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 31 identify_splits_and_merges 1.es4 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 31 identify_splits_and_merges 1.7s3 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 31 identify_splits_and_merges 1.fs1 e30 to e31 pg_nums {18=16} 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 31 prune_pg_created 2019-12-07T19:34:08.601+0000 7fe2d7285700 10 osd.0:0.consume_map 31 (was 30) 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 2019-12-07T19:34:08.601+0000 7fe2d7285700 10 osd.0:1.consume_map 31 (was 30) 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:08.601+0000 7fe2d7285700 10 osd.0:2.consume_map 31 (was 30) 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:08.601+0000 7fe2d7285700 10 osd.0:3.consume_map 31 (was 30) 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 2019-12-07T19:34:08.601+0000 7fe2d7285700 10 osd.0:4.consume_map 31 (was 30) 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:08.601+0000 7fe2d7285700 10 osd.0:5.consume_map 31 (was 30) 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:08.601+0000 7fe2d7285700 10 osd.0:6.consume_map 31 (was 30) 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:08.601+0000 7fe2d7285700 10 osd.0:7.consume_map 31 (was 30) 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:08.601+0000 7fe2d7285700 10 osd.0 31 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:08.601+0000 7fe2d7285700 15 osd.0 31 enqueue_peering_evt 1.8s5 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7fe2d7285700 15 osd.0 31 enqueue_peering_evt 1.9s3 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7fe2d7285700 15 osd.0 31 enqueue_peering_evt 1.1s1 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7fe2d7285700 15 osd.0 31 enqueue_peering_evt 1.as3 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.601+0000 7fe2d7285700 15 osd.0 31 enqueue_peering_evt 1.2s0 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558e8192b000 2019-12-07T19:34:08.601+0000 7fe2d7285700 15 osd.0 31 enqueue_peering_evt 1.3s5 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7fe2d7285700 15 osd.0 31 enqueue_peering_evt 1.bs1 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7fe2d7285700 15 osd.0 31 enqueue_peering_evt 1.4s1 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7fe2d7285700 15 osd.0 31 enqueue_peering_evt 1.5s4 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7fe2d7285700 15 osd.0 31 enqueue_peering_evt 1.ds5 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7fe2d7285700 15 osd.0 31 enqueue_peering_evt 1.es4 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7fe2d7285700 15 osd.0 31 enqueue_peering_evt 1.7s3 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 30 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.9s3 30 -> 31 2019-12-07T19:34:08.601+0000 7fe2d7285700 15 osd.0 31 enqueue_peering_evt 1.fs1 epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) 2019-12-07T19:34:08.601+0000 7fe2d7285700 7 osd.0 31 activate_map version 31 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 30 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.601+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.601+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.601+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.601+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558e81938000 2019-12-07T19:34:08.601+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.601+0000 7fe2c5a62700 10 osd.0 pg_epoch: 30 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.601+0000 7fe2c5a62700 20 osd.0:4.update_pg_epoch 1.4s1 30 -> 31 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.601+0000 7fe2c5a62700 10 osd.0 pg_epoch: 30 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:08.601+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.601+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.601+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.601+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558e8193b000 2019-12-07T19:34:08.601+0000 7fe2c5a62700 10 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.601+0000 7fe2c5a62700 10 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.601+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 osd.0 pg_epoch: 30 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.601+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.as3 30 -> 31 2019-12-07T19:34:08.601+0000 7fe2c5a62700 10 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 osd.0 pg_epoch: 30 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,4,2,0,5,3]/[1,4,2,0,5,3] -- 1/1 2019-12-07T19:34:08.601+0000 7fe2c5a62700 20 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.601+0000 7fe2c5a62700 20 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.601+0000 7fe2c5a62700 20 osd.0 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.601+0000 7fe2c5a62700 10 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.601+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:34:08.601+0000 7fe2c5a62700 10 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.601+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.601+0000 7fe2c5a62700 20 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 31 2019-12-07T19:34:08.601+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:08.601+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.601+0000 7fe2caa6c700 20 osd.0 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.601+0000 7fe2c5a62700 10 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.601+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:08.601+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.601+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.601+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:08.601+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.601+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.601+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558e81a98000 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:08.601+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.601+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.601+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.601+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558e81075000 2019-12-07T19:34:08.601+0000 7fe2c8a68700 10 osd.0 pg_epoch: 30 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.601+0000 7fe2c8a68700 20 osd.0:6.update_pg_epoch 1.es4 30 -> 31 2019-12-07T19:34:08.601+0000 7fe2c8a68700 10 osd.0 pg_epoch: 30 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,3,4,5,0,2]/[1,3,4,5,0,2] -- 1/1 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.601+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.2s0 30 -> 31 2019-12-07T19:34:08.601+0000 7fe2c8a68700 10 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 osd.0 pg_epoch: 30 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:08.601+0000 7fe2c8a68700 10 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.601+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[2~1,4~1]] state: Active advmap 2019-12-07T19:34:08.601+0000 7fe2c8a68700 10 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.601+0000 7fe2c8a68700 20 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[2~1,4~1]] on_active_advmap new removed_snaps [7~1], snap_trimq now [7~1] 2019-12-07T19:34:08.601+0000 7fe2c8a68700 20 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.601+0000 7fe2c8a68700 20 osd.0 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:08.601+0000 7fe2c8a68700 10 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:08.601+0000 7fe2c8a68700 10 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.601+0000 7fe2c8a68700 20 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558e819e4000 2019-12-07T19:34:08.601+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:08.601+0000 7fe2c8a68700 10 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.601+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 30 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.1s1 30 -> 31 2019-12-07T19:34:08.601+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:08.601+0000 7fe2cc26f700 20 osd.0 31 agent_entry empty queue 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 30 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] handle_advance_map [3,0,2,4,5,1]/[3,0,2,4,5,1] -- 3/3 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.601+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.601+0000 7fe2cb26d700 20 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:08.601+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.602+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.601+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] on_active_advmap new purged_snaps [2~1], now [4~1] 2019-12-07T19:34:08.602+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] share_pg_info 2019-12-07T19:34:08.602+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558e841a4000 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558e8404d100 con 0x558e818d7c00 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558e83d07000 2019-12-07T19:34:08.602+0000 7fe2cba6e700 10 osd.0 pg_epoch: 30 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.602+0000 7fe2cba6e700 20 osd.0:0.update_pg_epoch 1.8s5 30 -> 31 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 30 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0:5.update_pg_epoch 1.5s4 30 -> 31 2019-12-07T19:34:08.602+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558e80ff3100 con 0x558e818d7000 2019-12-07T19:34:08.602+0000 7fe2cba6e700 10 osd.0 pg_epoch: 30 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,4,5,2,3,0]/[1,4,5,2,3,0] -- 1/1 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 30 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] handle_advance_map [2,1,3,4,0,5]/[2,1,3,4,0,5] -- 2/2 2019-12-07T19:34:08.602+0000 7fe2cba6e700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.602+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558e83c9f100 con 0x558e818fe400 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.602+0000 7fe2cba6e700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.602+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.8s5_head options 2019-12-07T19:34:08.602+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.5s4_head options 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.602+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558e83c9e300 con 0x558e818d6800 2019-12-07T19:34:08.602+0000 7fe2cba6e700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.602+0000 7fe2cba6e700 20 osd.0 pg_epoch: 31 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.602+0000 7fe2cba6e700 20 osd.0 pg_epoch: 31 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.602+0000 7fe2cba6e700 20 osd.0 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.602+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558e81229c00 con 0x558e818fe000 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.602+0000 7fe2cba6e700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.602+0000 7fe2cba6e700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.602+0000 7fe2cba6e700 20 osd.0 pg_epoch: 31 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.602+0000 7fe2cba6e700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.602+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2caa6c700 15 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] publish_stats_to_osd 29:89 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558e819e7000 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] state: Started advmap 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.602+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 30 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0:5.update_pg_epoch 1.ds5 30 -> 31 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] plpg_on_pool_change 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] hit_set_clear 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 30 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] agent_stop 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] handle_activate_map: 31 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.602+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] activate_map 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] state: Active: handling ActMap 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] Active: kicking snap trim 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] exit NotTrimming 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] enter Trimming 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 28 while current is 31 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] enter Trimming/WaitReservation 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.602+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2caa6c700 15 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] publish_stats_to_osd 30:90 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558e81231000 2019-12-07T19:34:08.602+0000 7fe2caa6c700 7 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] state: handle ActMap primary 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.602+0000 7fe2caa6c700 15 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] publish_stats_to_osd 31:91 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 30 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.3s5 30 -> 31 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 30 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:08.602+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.3s5_head options 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e8417b800 seq 38 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.602+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417b800 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417b800 onodes shared_blobs 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417b800 allocated 0x[] released 0x[] 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 prepare 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417b800 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 io_done 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558e83c78000 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 30 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs1 30 -> 31 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 30 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.602+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs1_head options 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.602+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x116007~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.602+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x116007 0x116007~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe6007 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x7 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x47c and padding block with 0xb84 2019-12-07T19:34:08.602+0000 7fe2cea74700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:08.602+0000 7fe2cea74700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] SnapTrimmer state: NotTrimming: trimming 7 2019-12-07T19:34:08.602+0000 7fe2cea74700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[7~1] ps=[4~1]] exit Trimming/WaitReservation 2019-12-07T19:34:08.602+0000 7fe2cea74700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.602+0000 7fe2cea74700 15 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] publish_stats_to_osd 31:92 2019-12-07T19:34:08.602+0000 7fe2cea74700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:08.602+0000 7fe2cea74700 10 osd.0 31 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[7~1] ps=[4~1]] for snaptrim 2019-12-07T19:34:08.602+0000 7fe2cea74700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=31) prio 5 cost 1048576 e31) 2019-12-07T19:34:08.602+0000 7fe2cea74700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[7~1] ps=[4~1]] prepare_stats_for_publish reporting purged_snaps [4~1] 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11647c 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.602+0000 7fe2cea74700 15 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[7~1] ps=[4~1]] publish_stats_to_osd 31:93 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000127406s (0.000000097s flush + 0.000127309s kv commit) 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.602+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.602+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417b800> 2019-12-07T19:34:08.602+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.602+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 kv_submitted 2019-12-07T19:34:08.602+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417b800 2019-12-07T19:34:08.602+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b800 finishing 2019-12-07T19:34:08.602+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417b800 onodes 2019-12-07T19:34:08.602+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b800 done 2019-12-07T19:34:08.602+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:08.602+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417b800 [] 2019-12-07T19:34:08.602+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=31) prio 5 cost 1048576 e31) queued 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=31) prio 5 cost 1048576 e31) pg 0x558e81075000 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[7~1] ps=[4~1]] snap_trimmer posting 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[7~1] ps=[4~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 7 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 1 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore.OmapIteratorImpl(0x558e8405ef00) upper_bound after SNA_1_0000000000000007_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_0000000000000007_.0_0000000000000001.2' 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore.OmapIteratorImpl(0x558e8405ef00) valid is at 0x00000000000000000000000000000402'~' 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000007_.0_0000000000000001.2) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[7~1] ps=[4~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1]] SnapTrimmer state: adding snap 7 to purged_snaps 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1,7~1]] SnapTrimmer state: purged_snaps now [4~1,7~1], snap_trimq now [] 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e8417b200 seq 39 2019-12-07T19:34:08.602+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417b200 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417b200 onodes shared_blobs 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417b200 allocated 0x[] released 0x[] 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 prepare 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417b200 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 io_done 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1,7~1]] share_pg_info 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558e8122e000 2019-12-07T19:34:08.602+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 30 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.7s3 30 -> 31 2019-12-07T19:34:08.602+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558e83c47c00 con 0x558e818d7000 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 30 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:08.602+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558e83c46e00 con 0x558e818fe400 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.602+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:34:08.602+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558e83c46700 con 0x558e818d6800 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.602+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 -- 0x558e84704000 con 0x558e818fe000 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1,7~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1,7~1]] exit Trimming 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 NullEvt) prio 255 cost 10 e31) pg 0x558e81a9b000 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 30 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] handle_advance_map: 31 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.fs1 30 -> 31 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 30 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] handle_advance_map [3,0,5,4,1,2]/[3,0,5,4,1,2] -- 3/3 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] state: Started advmap 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:08.602+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] plpg_on_pool_change 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] hit_set_clear 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] agent_stop 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 31 advance_pg new pool opts old pool opts 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11647c~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.602+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11647c 0x11647c~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] handle_activate_map: 31 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe647c 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x47c 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] activate_map 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.602+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x8e9 and padding block with 0x717 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] activate_map: Not dirtying info: last_persisted is 29 while current is 31 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 NullEvt 2019-12-07T19:34:08.602+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:08.602+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.602+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:08.603+0000 7fe2caa6c700 15 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,7~1]] publish_stats_to_osd 31:94 2019-12-07T19:34:08.603+0000 7fe2caa6c700 20 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,7~1]] enter NotTrimming 2019-12-07T19:34:08.603+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,7~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:08.603+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,7~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:08.603+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,7~1]] snap_trimmer complete 2019-12-07T19:34:08.603+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.603+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.604+0000 7fe2e4535700 10 osd.0 31 tick 2019-12-07T19:34:08.604+0000 7fe2e4535700 10 osd.0 31 do_waiters -- start 2019-12-07T19:34:08.604+0000 7fe2e4535700 10 osd.0 31 do_waiters -- finish 2019-12-07T19:34:08.604+0000 7fe2e4535700 20 osd.0 31 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:08.605+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1168e9 2019-12-07T19:34:08.605+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.605+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.605+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.605+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.605+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.002239604s (0.000000110s flush + 0.002239494s kv commit) 2019-12-07T19:34:08.605+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.605+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.605+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417b200> 2019-12-07T19:34:08.605+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.605+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 kv_submitted 2019-12-07T19:34:08.605+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417b200 2019-12-07T19:34:08.605+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b200 finishing 2019-12-07T19:34:08.605+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417b200 onodes 2019-12-07T19:34:08.605+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b200 done 2019-12-07T19:34:08.605+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:08.605+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417b200 [] 2019-12-07T19:34:08.605+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.605+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.605+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.605+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.606+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 43 ==== pg_info2(1.ds5 1.ds5 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e84704000 con 0x558e818d6000 2019-12-07T19:34:08.606+0000 7fe2e753b700 15 osd.0 31 enqueue_peering_evt 1.ds5 epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:08.606+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.606+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 44 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e84704000 con 0x558e818d6000 2019-12-07T19:34:08.606+0000 7fe2e753b700 15 osd.0 31 enqueue_peering_evt 1.5s4 epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:08.606+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) pg 0x558e819e7000 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e845d4300 seq 18 2019-12-07T19:34:08.606+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845d4300 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845d4300 onodes shared_blobs 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845d4300 allocated 0x[] released 0x[] 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4300 prepare 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845d4300 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4300 io_done 2019-12-07T19:34:08.606+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.606+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.606+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e31) pg 0x558e83d07000 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:08.606+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:34:08.606+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.606+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.606+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.606+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1168e9~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.606+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1168e9 0x1168e9~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:08.606+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.606+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe68e9 2019-12-07T19:34:08.606+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x8e9 2019-12-07T19:34:08.606+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.606+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xd5e and padding block with 0x2a2 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e845d4600 seq 16 2019-12-07T19:34:08.606+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._biginfo' <- _biginfo 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._epoch' <- _epoch 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845d4600 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845d4600 onodes shared_blobs 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845d4600 allocated 0x[] released 0x[] 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4600 prepare 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845d4600 2019-12-07T19:34:08.606+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4600 io_done 2019-12-07T19:34:08.606+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.606+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 45 ==== pg_info2(1.ds5 1.ds5 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e84704000 con 0x558e818d6000 2019-12-07T19:34:08.606+0000 7fe2e753b700 15 osd.0 31 enqueue_peering_evt 1.ds5 epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:08.606+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.606+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.606+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.606+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.606+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e31) pg 0x558e819e7000 2019-12-07T19:34:08.606+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:08.606+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.606+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:08.607+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e819a1b00 seq 19 2019-12-07T19:34:08.607+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:08.607+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 46 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e84704000 con 0x558e818d6000 2019-12-07T19:34:08.607+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:34:08.607+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1b00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1b00 onodes shared_blobs 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1b00 allocated 0x[] released 0x[] 2019-12-07T19:34:08.607+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 prepare 2019-12-07T19:34:08.607+0000 7fe2e753b700 15 osd.0 31 enqueue_peering_evt 1.5s4 epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1b00 2019-12-07T19:34:08.607+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 io_done 2019-12-07T19:34:08.607+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e31) pg 0x558e83d07000 2019-12-07T19:34:08.607+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:08.607+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.607+0000 7fe2c5261700 10 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.607+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:08.607+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e84558f00 seq 17 2019-12-07T19:34:08.607+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._biginfo' <- _biginfo 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:08.607+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:08.607+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84558f00 cost 5443 (1 ios * 4000 + 1443 bytes) 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84558f00 onodes shared_blobs 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84558f00 allocated 0x[] released 0x[] 2019-12-07T19:34:08.607+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558f00 prepare 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84558f00 2019-12-07T19:34:08.607+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558f00 io_done 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.607+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x116d5e 2019-12-07T19:34:08.607+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.607+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.607+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.607+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.607+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.001129486s (0.000000103s flush + 0.001129383s kv commit) 2019-12-07T19:34:08.607+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:08.607+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1288 < min_flush_size 524288 2019-12-07T19:34:08.607+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 2 2019-12-07T19:34:08.607+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2398 < min_flush_size 524288 2019-12-07T19:34:08.607+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:34:08.607+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3678 < min_flush_size 524288 2019-12-07T19:34:08.607+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:08.607+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3701 < min_flush_size 524288 2019-12-07T19:34:08.607+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3701 < min_flush_size 524288 2019-12-07T19:34:08.607+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.607+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x116d5e~e75 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.607+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x116d5e 0x116d5e~e75 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.607+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.607+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe6d5e 2019-12-07T19:34:08.607+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xd5e 2019-12-07T19:34:08.607+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.607+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xbd3 and padding block with 0x42d 2019-12-07T19:34:08.607+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.607+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e845d4300> 2019-12-07T19:34:08.607+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.607+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4300 kv_submitted 2019-12-07T19:34:08.607+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845d4300 2019-12-07T19:34:08.607+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4300 finishing 2019-12-07T19:34:08.607+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845d4300 onodes 2019-12-07T19:34:08.607+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845d4300 done 2019-12-07T19:34:08.607+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1b00 kv_submitted 2019-12-07T19:34:08.607+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845d4300 [] 2019-12-07T19:34:08.607+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.607+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.607+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.607+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.607+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x117bd3 2019-12-07T19:34:08.608+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.608+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.608+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.608+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.608+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 3 cleaned 0 in 0.000183541s (0.000000090s flush + 0.000183451s kv commit) 2019-12-07T19:34:08.608+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e845d4600,0x558e819a1b00,0x558e84558f00> 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.608+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4600 kv_submitted 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845d4600 2019-12-07T19:34:08.608+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4600 finishing 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845d4600 onodes 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845d4600 done 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84558f00 kv_submitted 2019-12-07T19:34:08.608+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845d4600 [] 2019-12-07T19:34:08.608+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.608+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 kv_submitted 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1b00 2019-12-07T19:34:08.608+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 finishing 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1b00 onodes 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1b00 done 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:34:08.608+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1b00 [] 2019-12-07T19:34:08.608+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.608+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558f00 kv_submitted 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84558f00 2019-12-07T19:34:08.608+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558f00 finishing 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84558f00 onodes 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84558f00 done 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:08.608+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84558f00 [] 2019-12-07T19:34:08.608+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.608+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.608+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.608+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:08.608+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 121 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83c46e00 con 0x558e818fe400 2019-12-07T19:34:08.608+0000 7fe2e6d3a700 15 osd.0 31 enqueue_peering_evt 1.bs1 epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:08.608+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) pg 0x558e83c78000 2019-12-07T19:34:08.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:08.608+0000 7fe2c6263700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.608+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:08.608+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:08.608+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e8106c300 seq 14 2019-12-07T19:34:08.608+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:08.608+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._biginfo' <- _biginfo 2019-12-07T19:34:08.608+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._epoch' <- _epoch 2019-12-07T19:34:08.608+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:08.608+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:08.608+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8106c300 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.608+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8106c300 onodes shared_blobs 2019-12-07T19:34:08.608+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8106c300 allocated 0x[] released 0x[] 2019-12-07T19:34:08.608+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 prepare 2019-12-07T19:34:08.608+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8106c300 2019-12-07T19:34:08.608+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 io_done 2019-12-07T19:34:08.608+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.608+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.608+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1069 < min_flush_size 524288 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1129 < min_flush_size 524288 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1152 < min_flush_size 524288 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1152 < min_flush_size 524288 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x117bd3~480 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x117bd3 0x117bd3~480 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe7bd3 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xbd3 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x53 and padding block with 0xfad 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x118053 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000139908s (0.000000096s flush + 0.000139812s kv commit) 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8106c300> 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 kv_submitted 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8106c300 2019-12-07T19:34:08.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8106c300 finishing 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8106c300 onodes 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8106c300 done 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:08.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8106c300 [] 2019-12-07T19:34:08.609+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.609+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.609+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 122 ==== pg_info2(1.3s5 1.3s5 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83c46e00 con 0x558e818fe400 2019-12-07T19:34:08.609+0000 7fe2e6d3a700 15 osd.0 31 enqueue_peering_evt 1.3s5 epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:08.609+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.609+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.609+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.609+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.609+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) pg 0x558e81231000 2019-12-07T19:34:08.609+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:08.609+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.609+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:08.609+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:08.609+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e845c3800 seq 14 2019-12-07T19:34:08.609+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:08.609+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._biginfo' <- _biginfo 2019-12-07T19:34:08.609+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._epoch' <- _epoch 2019-12-07T19:34:08.609+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:08.609+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:08.609+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845c3800 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.609+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845c3800 onodes shared_blobs 2019-12-07T19:34:08.609+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845c3800 allocated 0x[] released 0x[] 2019-12-07T19:34:08.609+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c3800 prepare 2019-12-07T19:34:08.609+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845c3800 2019-12-07T19:34:08.609+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c3800 io_done 2019-12-07T19:34:08.609+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x118053~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x118053 0x118053~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe8053 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x53 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x4c8 and padding block with 0xb38 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1184c8 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.609+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000105592s (0.000000080s flush + 0.000105512s kv commit) 2019-12-07T19:34:08.609+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e845c3800> 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c3800 kv_submitted 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845c3800 2019-12-07T19:34:08.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845c3800 finishing 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845c3800 onodes 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845c3800 done 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:08.609+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845c3800 [] 2019-12-07T19:34:08.609+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.609+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.609+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.609+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 124 ==== MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=56, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8417c000 con 0x558e818fe400 2019-12-07T19:34:08.609+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 123 ==== osd_map(30..31 src has 1..31) v4 ==== 1286+0+0 (unknown 0 0 0) 0x558e84409d40 con 0x558e818fe400 2019-12-07T19:34:08.609+0000 7fe2d4a80700 20 osd.0 31 OSD::ms_dispatch: osd_map(30..31 src has 1..31) v4 2019-12-07T19:34:08.609+0000 7fe2d4a80700 10 osd.0 31 do_waiters -- start 2019-12-07T19:34:08.609+0000 7fe2d4a80700 10 osd.0 31 do_waiters -- finish 2019-12-07T19:34:08.609+0000 7fe2e6d3a700 15 osd.0 31 enqueue_op 0x558e83dd82c0 prio 127 cost 0 latency 0.000023 epoch 31 MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=56, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:08.609+0000 7fe2d4a80700 20 osd.0 31 _dispatch 0x558e84409d40 osd_map(30..31 src has 1..31) v4 2019-12-07T19:34:08.609+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=56, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) 2019-12-07T19:34:08.609+0000 7fe2d4a80700 3 osd.0 31 handle_osd_map epochs [30,31], i have 31, src has [1,31] 2019-12-07T19:34:08.609+0000 7fe2d4a80700 10 osd.0 31 no new maps here, dropping 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=56, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) queued 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=56, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) pg 0x558e83c78000 2019-12-07T19:34:08.609+0000 7fe2c6263700 10 osd.0 31 dequeue_op 0x558e83dd82c0 prio 127 cost 0 latency 0.000067 MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=56, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] 2019-12-07T19:34:08.609+0000 7fe2c6263700 10 osd.0 31 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 29 -> 31 (as per caller) 2019-12-07T19:34:08.609+0000 7fe2c6263700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] _handle_message: MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=56, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:08.609+0000 7fe2c6263700 25 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:08.609+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~32000 size 0x32000 (204800) 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014af0 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7b5e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a7b490 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84190f50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84190fc0 blob([0x43e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191030 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841910a0 blob([0x440000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191110 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191180 blob([0x442000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841911f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191260 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841912d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191340 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841913b0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191420 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191490 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191500 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191570 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841915e0 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191650 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841916c0 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191730 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841917a0 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191810 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191880 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841918f0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191960 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841919d0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191a40 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191ab0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191b20 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191b90 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191c00 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191c70 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191ce0 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191d50 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191dc0 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191e30 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191ea0 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191f10 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191f80 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a48000 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a48070 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a480e0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a48150 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a481c0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a48230 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a482a0 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a48310 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a48380 sbid 0x0)) need 0x0~2000 cache has 0x[] 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a48070 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a480e0 sbid 0x0)) need {<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2a000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a48150 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a481c0 sbid 0x0)) need {<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2c000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a48230 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a482a0 sbid 0x0)) need {<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2e000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a48310 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a48380 sbid 0x0)) need {<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x30000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e81a7b490 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84190f50 sbid 0x0)) need {<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x2000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84014af0 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7b5e0 sbid 0x0)) need {<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x0: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84190fc0 blob([0x43e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191030 sbid 0x0)) need {<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x4000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841910a0 blob([0x440000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191110 sbid 0x0)) need {<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x6000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84191180 blob([0x442000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841911f0 sbid 0x0)) need {<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x8000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84191260 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841912d0 sbid 0x0)) need {<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xa000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84191340 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841913b0 sbid 0x0)) need {<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xc000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84191420 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191490 sbid 0x0)) need {<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0xe000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84191500 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191570 sbid 0x0)) need {<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x10000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841915e0 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191650 sbid 0x0)) need {<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x12000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841916c0 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191730 sbid 0x0)) need {<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x14000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e841917a0 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191810 sbid 0x0)) need {<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x16000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84191880 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841918f0 sbid 0x0)) need {<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x18000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84191960 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841919d0 sbid 0x0)) need {<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1a000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84191a40 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191ab0 sbid 0x0)) need {<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1c000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84191b20 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191b90 sbid 0x0)) need {<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x1e000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84191c00 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191c70 sbid 0x0)) need {<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x20000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84191ce0 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191d50 sbid 0x0)) need {<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x22000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84191dc0 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191e30 sbid 0x0)) need {<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x24000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84191ea0 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191f10 sbid 0x0)) need {<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x26000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc blob Blob(0x558e84191f80 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a48000 sbid 0x0)) need {<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _prepare_read_ioc region 0x28000: 0x0 reading 0x0~2000 2019-12-07T19:34:08.609+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read waiting for aio 2019-12-07T19:34:08.610+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 125 ==== MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e8417cc80 con 0x558e818fe400 2019-12-07T19:34:08.610+0000 7fe2e6d3a700 15 osd.0 31 enqueue_op 0x558e83dd8580 prio 127 cost 0 latency 0.000021 epoch 31 MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:08.610+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) 2019-12-07T19:34:08.610+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 126 ==== MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=58, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e84314000 con 0x558e818fe400 2019-12-07T19:34:08.610+0000 7fe2e6d3a700 15 osd.0 31 enqueue_op 0x558e83dd8840 prio 127 cost 0 latency 0.000016 epoch 31 MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=58, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:08.610+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=58, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) 2019-12-07T19:34:08.610+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.610+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) queued 2019-12-07T19:34:08.610+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 127 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c46e00 con 0x558e818fe400 2019-12-07T19:34:08.610+0000 7fe2e6d3a700 15 osd.0 31 enqueue_peering_evt 1.bs1 epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:08.610+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a48070 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a480e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2a000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a480f8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843d2870 space 0x558e81a480f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a48150 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a481c0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2c000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a481d8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e81a46a20 space 0x558e81a481d8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a48230 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a482a0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2e000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a482b8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e81a46ab0 space 0x558e81a482b8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a48310 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a48380 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x30000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a48398 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e81a46b40 space 0x558e81a48398 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e81a7b490 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84190f50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x2000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84190f68 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e81a46bd0 space 0x558e84190f68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84014af0 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7b5e0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x0:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a7b5f8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e81a46c60 space 0x558e81a7b5f8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84190fc0 blob([0x43e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191030 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x4000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191048 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e81a46cf0 space 0x558e84191048 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841910a0 blob([0x440000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191110 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x6000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191128 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e81a46d80 space 0x558e84191128 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84191180 blob([0x442000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841911f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x8000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191208 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e81a46e10 space 0x558e84191208 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84191260 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841912d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xa000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841912e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e81a46ea0 space 0x558e841912e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84191340 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841913b0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xc000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841913c8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843d3440 space 0x558e841913c8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84191420 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191490 sbid 0x0)) need 0x{<0x0, 0x2000> : [0xe000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841914a8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843d3560 space 0x558e841914a8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84191500 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191570 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x10000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191588 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843d3320 space 0x558e84191588 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841915e0 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191650 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x12000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191668 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e83c8e630 space 0x558e84191668 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841916c0 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191730 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x14000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191748 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e83c8ecf0 space 0x558e84191748 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e841917a0 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191810 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x16000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191828 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e83c8ec60 space 0x558e84191828 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84191880 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841918f0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x18000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191908 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e83c8eab0 space 0x558e84191908 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84191960 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841919d0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1a000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e841919e8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e83c8e6c0 space 0x558e841919e8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84191a40 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191ab0 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1c000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191ac8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e83c8ebd0 space 0x558e84191ac8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84191b20 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191b90 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x1e000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191ba8 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e846da000 space 0x558e84191ba8 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84191c00 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191c70 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x20000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191c88 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843baab0 space 0x558e84191c88 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84191ce0 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191d50 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x22000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191d68 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843bab40 space 0x558e84191d68 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84191dc0 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191e30 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x24000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191e48 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843babd0 space 0x558e84191e48 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84191ea0 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191f10 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x26000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84191f28 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843baf30 space 0x558e84191f28 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _generate_read_result_bl blob Blob(0x558e84191f80 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a48000 sbid 0x0)) need 0x{<0x0, 0x2000> : [0x28000:0~2000]} 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a48018 in 0x558e80219c00) _discard 0x0~2000 2019-12-07T19:34:08.610+0000 7fe2c6263700 20 _add level 1 near 0 on buffer(0x558e843baea0 space 0x558e81a48018 0x0~2000 clean) which has cache_private 0 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 = 204800 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] handle_sub_read read request=204800 r=204800 len=204800 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] get_hash_info: Getting attr on 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] get_hash_info: not in cache 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] get_hash_info: found on disk, size 204800 2019-12-07T19:34:08.611+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# hinfo_key 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# hinfo_key = 0 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] handle_sub_read: Checking hash of 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.611+0000 7fe2c6263700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 31/28 ECSubReadReply(tid=56, attrs_read=0)) v2 -- 0x558e846c2c80 con 0x558e818fe400 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 osd.0 31 dequeue_op 0x558e83dd82c0 finish 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) pg 0x558e83c78000 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 osd.0 31 dequeue_op 0x558e83dd8580 prio 127 cost 0 latency 0.000827 MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] _handle_message: MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=57, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:08.611+0000 7fe2ca26b700 25 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:08.611+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~32000 size 0x32000 (204800) 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014af0 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7b5e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a7b490 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84190f50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84190fc0 blob([0x43e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191030 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841910a0 blob([0x440000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191110 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191180 blob([0x442000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841911f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191260 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841912d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191340 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841913b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191420 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191490 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191500 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191570 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841915e0 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191650 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841916c0 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191730 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841917a0 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191810 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191880 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841918f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191960 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841919d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191a40 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191ab0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191b20 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191b90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191c00 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191c70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191ce0 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191d50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191dc0 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191e30 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191ea0 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191f10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191f80 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a48000 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a48070 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a480e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a48150 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a481c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a48230 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a482a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a48310 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a48380 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 = 204800 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] handle_sub_read read request=204800 r=204800 len=204800 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] get_hash_info: Getting attr on 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] get_hash_info: not in cache 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] get_hash_info: found on disk, size 204800 2019-12-07T19:34:08.611+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# hinfo_key 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# hinfo_key = 0 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] handle_sub_read: Checking hash of 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.611+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 31/28 ECSubReadReply(tid=57, attrs_read=0)) v2 -- 0x558e84314780 con 0x558e818fe400 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 osd.0 31 dequeue_op 0x558e83dd8580 finish 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=58, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) queued 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e31) pg 0x558e83c78000 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e83c57500 seq 15 2019-12-07T19:34:08.611+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._biginfo' <- _biginfo 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c57500 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c57500 onodes shared_blobs 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c57500 allocated 0x[] released 0x[] 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 prepare 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c57500 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 io_done 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGOpItem(op=MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=58, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e31) pg 0x558e83c78000 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1184c8~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1184c8 0x1184c8~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe84c8 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x4c8 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 osd.0 31 dequeue_op 0x558e83dd8840 prio 127 cost 0 latency 0.001031 MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=58, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x935 and padding block with 0x6cb 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] _handle_message: MOSDECSubOpRead(1.bs1 31/28 ECSubRead(tid=58, to_read={1:d313ddd1:::smithi13140796-49:head=0,204800,0}, subchunks={1:d313ddd1:::smithi13140796-49:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:08.611+0000 7fe2ca26b700 25 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:08.611+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~32000 size 0x32000 (204800) 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84014af0 blob([0x43a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7b5e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a7b490 blob([0x43c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84190f50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84190fc0 blob([0x43e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191030 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841910a0 blob([0x440000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191110 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191180 blob([0x442000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841911f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191260 blob([0x444000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841912d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x118935 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191340 blob([0x446000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841913b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191420 blob([0x448000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191490 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191500 blob([0x44a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191570 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841915e0 blob([0x44c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191650 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841916c0 blob([0x44e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191730 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000089923s (0.000000083s flush + 0.000089840s kv commit) 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841917a0 blob([0x450000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191810 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191880 blob([0x452000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841918f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191960 blob([0x454000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841919d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191a40 blob([0x456000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191ab0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191b20 blob([0x458000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191b90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191c00 blob([0x45a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191c70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191ce0 blob([0x45c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191d50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191dc0 blob([0x45e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191e30 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191ea0 blob([0x460000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84191f10 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84191f80 blob([0x462000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a48000 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a48070 blob([0x464000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a480e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a48150 blob([0x466000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a481c0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a48230 blob([0x468000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a482a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a48310 blob([0x46a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a48380 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 0x0~32000 = 204800 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_sub_read read request=204800 r=204800 len=204800 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] get_hash_info: Getting attr on 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] get_hash_info: not in cache 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] get_hash_info: found on disk, size 204800 2019-12-07T19:34:08.611+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# hinfo_key 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.bs1_head 1#1:d313ddd1:::smithi13140796-49:head# hinfo_key = 0 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_sub_read: Checking hash of 1:d313ddd1:::smithi13140796-49:head 2019-12-07T19:34:08.611+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpReadReply(1.bs0 31/28 ECSubReadReply(tid=58, attrs_read=0)) v2 -- 0x558e8417cc80 con 0x558e818fe400 2019-12-07T19:34:08.611+0000 7fe2ca26b700 10 osd.0 31 dequeue_op 0x558e83dd8840 finish 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c57500> 2019-12-07T19:34:08.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 kv_submitted 2019-12-07T19:34:08.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c57500 2019-12-07T19:34:08.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57500 finishing 2019-12-07T19:34:08.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c57500 onodes 2019-12-07T19:34:08.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57500 done 2019-12-07T19:34:08.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:08.611+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c57500 [] 2019-12-07T19:34:08.611+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.611+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 128 ==== pg_info2(1.3s5 1.3s5 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c46e00 con 0x558e818fe400 2019-12-07T19:34:08.611+0000 7fe2e6d3a700 15 osd.0 31 enqueue_peering_evt 1.3s5 epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:08.611+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e31) pg 0x558e81231000 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e83c56f00 seq 15 2019-12-07T19:34:08.611+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._biginfo' <- _biginfo 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c56f00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c56f00 onodes shared_blobs 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c56f00 allocated 0x[] released 0x[] 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 prepare 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c56f00 2019-12-07T19:34:08.611+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 io_done 2019-12-07T19:34:08.611+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x118935~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.611+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x118935 0x118935~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe8935 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x935 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.611+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xda2 and padding block with 0x25e 2019-12-07T19:34:08.612+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 38 ==== pg_info2(1.4s1 1.4s1 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e84704000 con 0x558e818fe000 2019-12-07T19:34:08.612+0000 7fe2e753b700 15 osd.0 31 enqueue_peering_evt 1.4s1 epoch_sent: 31 epoch_requested: 31 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s) 2019-12-07T19:34:08.612+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.612+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x118da2 2019-12-07T19:34:08.612+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.612+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.612+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.612+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000246804s (0.000000091s flush + 0.000246713s kv commit) 2019-12-07T19:34:08.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c56f00> 2019-12-07T19:34:08.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 kv_submitted 2019-12-07T19:34:08.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c56f00 2019-12-07T19:34:08.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c56f00 finishing 2019-12-07T19:34:08.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c56f00 onodes 2019-12-07T19:34:08.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c56f00 done 2019-12-07T19:34:08.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:08.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c56f00 [] 2019-12-07T19:34:08.612+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.612+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.612+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:08.612+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 39 ==== pg_info2(1.4s1 1.4s1 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e84704000 con 0x558e818fe000 2019-12-07T19:34:08.612+0000 7fe2e753b700 15 osd.0 31 enqueue_peering_evt 1.4s1 epoch_sent: 31 epoch_requested: 31 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s) 2019-12-07T19:34:08.612+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.617+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 226 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e31) pg 0x558e81938000 2019-12-07T19:34:08.617+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.8s5 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.8s0( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:08.617+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.8s0( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s) 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:08.617+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 227 ==== pg_info2(1.as3 1.as3 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e83c57200 seq 11 2019-12-07T19:34:08.617+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.as3 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:08.617+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c57200 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c57200 onodes shared_blobs 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c57200 allocated 0x[] released 0x[] 2019-12-07T19:34:08.617+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 228 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57200 prepare 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c57200 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57200 io_done 2019-12-07T19:34:08.617+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.es4 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:08.617+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e31) pg 0x558e81938000 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s) 2019-12-07T19:34:08.617+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 229 ==== pg_info2(1.7s3 1.7s3 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.617+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.7s3 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:08.617+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e83c57b00 seq 12 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x118da2~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x118da2 0x118da2~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c57b00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe8da2 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c57b00 onodes shared_blobs 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xda2 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c57b00 allocated 0x[] released 0x[] 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x217 and padding block with 0xde9 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57b00 prepare 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c57b00 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57b00 io_done 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x119217 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000117217s (0.000000112s flush + 0.000117105s kv commit) 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c57200> 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57200 kv_submitted 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c57200 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57200 finishing 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c57200 onodes 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57200 done 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57b00 kv_queued 2019-12-07T19:34:08.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c57200 [] 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.8s0( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.617+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:08.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.8s0( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e31) pg 0x558e841a4000 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x119217~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x119217 0x119217~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe9217 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x217 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x684 and padding block with 0x97c 2019-12-07T19:34:08.617+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.8s0( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:08.617+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.617+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:08.617+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:08.617+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e819a1200 seq 33 2019-12-07T19:34:08.617+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:08.617+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._biginfo' <- _biginfo 2019-12-07T19:34:08.617+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._epoch' <- _epoch 2019-12-07T19:34:08.617+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:08.617+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:08.617+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.617+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes shared_blobs 2019-12-07T19:34:08.617+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[] released 0x[] 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x119684 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.617+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.617+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:34:08.617+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:34:08.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000102079s (0.000000075s flush + 0.000102004s kv commit) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x119684~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x119684 0x119684~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe9684 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x684 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xaf9 and padding block with 0x507 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c57b00> 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57b00 kv_submitted 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c57b00 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57b00 finishing 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c57b00 onodes 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57b00 done 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df900 q now empty 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c57b00 [] 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.617+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:08.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x119af9 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000232350s (0.000000063s flush + 0.000232287s kv commit) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1200> 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.617+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e31) pg 0x558e8193b000 2019-12-07T19:34:08.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:08.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.617+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:08.617+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:08.617+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e8417bb00 seq 21 2019-12-07T19:34:08.617+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:08.617+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:08.617+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:34:08.617+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:08.617+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:08.617+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417bb00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.617+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417bb00 onodes shared_blobs 2019-12-07T19:34:08.617+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:08.617+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417bb00 prepare 2019-12-07T19:34:08.617+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417bb00 2019-12-07T19:34:08.617+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417bb00 io_done 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x119af9~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x119af9 0x119af9~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe9af9 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xaf9 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xf6e and padding block with 0x92 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x119f6e 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000125796s (0.000000089s flush + 0.000125707s kv commit) 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417bb00> 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417bb00 kv_submitted 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417bb00 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417bb00 finishing 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417bb00 onodes 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417bb00 done 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417bb00 [] 2019-12-07T19:34:08.617+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.617+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.617+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.617+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.617+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.617+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e31) pg 0x558e8122e000 2019-12-07T19:34:08.617+0000 7fe2c8267700 10 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:08.617+0000 7fe2c8267700 10 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.617+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:08.617+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:08.617+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e819a0300 seq 35 2019-12-07T19:34:08.617+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:08.617+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:08.617+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:34:08.617+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:08.617+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:08.617+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0300 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.617+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0300 onodes shared_blobs 2019-12-07T19:34:08.617+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0300 allocated 0x[] released 0x[] 2019-12-07T19:34:08.617+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0300 prepare 2019-12-07T19:34:08.617+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0300 2019-12-07T19:34:08.617+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0300 io_done 2019-12-07T19:34:08.617+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:08.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.618+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.618+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x119f6e~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.618+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x119f6e 0x119f6e~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.618+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.618+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xe9f6e 2019-12-07T19:34:08.618+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xf6e 2019-12-07T19:34:08.618+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.618+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x3e3 and padding block with 0xc1d 2019-12-07T19:34:08.618+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11a3e3 2019-12-07T19:34:08.618+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.618+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.618+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.618+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000141062s (0.000000052s flush + 0.000141010s kv commit) 2019-12-07T19:34:08.618+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a0300> 2019-12-07T19:34:08.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0300 kv_submitted 2019-12-07T19:34:08.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0300 2019-12-07T19:34:08.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0300 finishing 2019-12-07T19:34:08.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0300 onodes 2019-12-07T19:34:08.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0300 done 2019-12-07T19:34:08.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:08.618+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0300 [] 2019-12-07T19:34:08.618+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.618+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.618+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.618+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e31) pg 0x558e81a98000 2019-12-07T19:34:08.619+0000 7fe2c4a60700 10 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:08.619+0000 7fe2c4a60700 10 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.619+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:08.619+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:08.619+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e8417a900 seq 25 2019-12-07T19:34:08.619+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:08.619+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:08.619+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:08.619+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:08.619+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:08.619+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417a900 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.619+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417a900 onodes shared_blobs 2019-12-07T19:34:08.619+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417a900 allocated 0x[] released 0x[] 2019-12-07T19:34:08.619+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a900 prepare 2019-12-07T19:34:08.619+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417a900 2019-12-07T19:34:08.619+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a900 io_done 2019-12-07T19:34:08.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:08.619+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.619+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.619+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.619+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:08.619+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.619+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.619+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.619+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11a3e3~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.619+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11a3e3 0x11a3e3~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.619+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.619+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xea3e3 2019-12-07T19:34:08.619+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x3e3 2019-12-07T19:34:08.619+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.619+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x858 and padding block with 0x7a8 2019-12-07T19:34:08.619+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11a858 2019-12-07T19:34:08.619+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.619+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.619+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.619+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.619+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000157455s (0.000000107s flush + 0.000157348s kv commit) 2019-12-07T19:34:08.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.619+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417a900> 2019-12-07T19:34:08.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a900 kv_submitted 2019-12-07T19:34:08.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417a900 2019-12-07T19:34:08.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a900 finishing 2019-12-07T19:34:08.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417a900 onodes 2019-12-07T19:34:08.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a900 done 2019-12-07T19:34:08.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:08.619+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417a900 [] 2019-12-07T19:34:08.619+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.619+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:08.619+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.619+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:08.620+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 230 ==== pg_info2(1.9s3 1.9s3 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:08.620+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.9s3 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:08.620+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.620+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 231 ==== osd_map(30..31 src has 1..31) v4 ==== 1286+0+0 (unknown 0 0 0) 0x558e846e2d80 con 0x558e818d7c00 2019-12-07T19:34:08.620+0000 7fe2d4a80700 20 osd.0 31 OSD::ms_dispatch: osd_map(30..31 src has 1..31) v4 2019-12-07T19:34:08.620+0000 7fe2d4a80700 10 osd.0 31 do_waiters -- start 2019-12-07T19:34:08.620+0000 7fe2d4a80700 10 osd.0 31 do_waiters -- finish 2019-12-07T19:34:08.620+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.620+0000 7fe2d4a80700 20 osd.0 31 _dispatch 0x558e846e2d80 osd_map(30..31 src has 1..31) v4 2019-12-07T19:34:08.620+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.620+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.620+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e31) pg 0x558e8192b000 2019-12-07T19:34:08.620+0000 7fe2d4a80700 3 osd.0 31 handle_osd_map epochs [30,31], i have 31, src has [1,31] 2019-12-07T19:34:08.620+0000 7fe2d4a80700 10 osd.0 31 no new maps here, dropping 2019-12-07T19:34:08.620+0000 7fe2c7265700 10 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:08.620+0000 7fe2c7265700 10 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.620+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:08.620+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:08.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83d4c300 seq 32 2019-12-07T19:34:08.620+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:08.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:08.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:08.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:08.620+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:08.620+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c300 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c300 onodes shared_blobs 2019-12-07T19:34:08.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c300 allocated 0x[] released 0x[] 2019-12-07T19:34:08.620+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 232 ==== MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=149, reqid=client.4540.0:261, at_version=31'30, trim_to=0'0, roll_forward_to=29'29)) v2 ==== 198580+0+0 (unknown 0 0 0) 0x558e83aca000 con 0x558e818d7c00 2019-12-07T19:34:08.620+0000 7fe2e7d3c700 15 osd.0 31 enqueue_op 0x558e8477b340 prio 127 cost 0 latency 0.000024 epoch 31 MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=149, reqid=client.4540.0:261, at_version=31'30, trim_to=0'0, roll_forward_to=29'29)) v2 2019-12-07T19:34:08.620+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=149, reqid=client.4540.0:261, at_version=31'30, trim_to=0'0, roll_forward_to=29'29)) v2) prio 127 cost 0 e31) 2019-12-07T19:34:08.620+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 prepare 2019-12-07T19:34:08.620+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c300 2019-12-07T19:34:08.620+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 io_done 2019-12-07T19:34:08.620+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=149, reqid=client.4540.0:261, at_version=31'30, trim_to=0'0, roll_forward_to=29'29)) v2) prio 127 cost 0 e31) queued 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=149, reqid=client.4540.0:261, at_version=31'30, trim_to=0'0, roll_forward_to=29'29)) v2) prio 127 cost 0 e31) pg 0x558e841a4000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 osd.0 31 dequeue_op 0x558e8477b340 prio 127 cost 0 latency 0.000730 MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=149, reqid=client.4540.0:261, at_version=31'30, trim_to=0'0, roll_forward_to=29'29)) v2 pg pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[4~1]] 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 osd.0 31 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 29 -> 31 (as per caller) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[4~1]] _handle_message: MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=149, reqid=client.4540.0:261, at_version=31'30, trim_to=0'0, roll_forward_to=29'29)) v2 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 29'29 (0'0,29'29] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[4~1]] append_log log((0'0,29'29], crt=29'29) [31'30 (0'0) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:261 2019-12-07T19:34:08.612951+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 1, new_object: 0] 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'30 (0'0,31'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=29'29 crt=29'29 lcod 29'29 active mbc={} ps=[4~1]] add_log_entry 31'30 (0'0) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:261 2019-12-07T19:34:08.612951+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'30 (0'0,31'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=29'29 crt=29'29 lcod 29'29 active mbc={} ps=[4~1]] append_log approx pg log length = 30 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'30 (0'0,31'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=29'29 crt=29'29 lcod 29'29 active mbc={} ps=[4~1]] append_log transaction_applied = 1 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:08.621+0000 7fe2cba6e700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 31'30, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e83e73500 seq 34 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.8s5_head 0x558e8403ed20) get_onode oid 5#1:14fc029d:::smithi13140796-41:head# key 0x85800000000000000114fc029d21736d'ithi13140796-41!='0xfffffffffffffffeffffffffffffffff'o' 2019-12-07T19:34:08.621+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1153 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# = 0 2019-12-07T19:34:08.621+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 2 keys 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 2 keys = 0 2019-12-07T19:34:08.621+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:08.621+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 0x0~30000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#1:14fc029d:::smithi13140796-41:head# 0x0~30000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~30000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e73500 24 blobs 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 fbmap_alloc 0x558e80233300 allocate 0x30000/1000,30000,0 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x7c000~f000/1000,30000,0 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x1c3000~21000/1000,30000,0 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x7c000~f000,0x1c3000~21000] 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c4f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c4f50 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4e00 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c4f50) get_ref 0x0~2000 Blob(0x558e846c4f50 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4e00 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c4f50) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e846c4f50 blob([0x7c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4e00 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83ac4e18 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c5650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846c56c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c5650 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c56c0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c5650) get_ref 0x0~2000 Blob(0x558e846c5650 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c56c0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c5650) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e846c5650 blob([0x7e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846c56c0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e846c56d8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c51f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846c5260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c51f0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c5260 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c51f0) get_ref 0x0~2000 Blob(0x558e846c51f0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c5260 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c51f0) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e846c51f0 blob([0x80000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846c5260 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e846c5278 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c4000 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846c4070 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c4000 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c4070 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c4000) get_ref 0x0~2000 Blob(0x558e846c4000 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c4070 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c4000) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e846c4000 blob([0x82000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846c4070 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e846c4088 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c40e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84190850 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c40e0 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84190850 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c40e0) get_ref 0x0~2000 Blob(0x558e846c40e0 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84190850 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c40e0) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e846c40e0 blob([0x84000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84190850 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e84190868 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84190310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edf730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11a858~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84190310 blob([0x86000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf730 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11a858 0x11a858~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e84190310) get_ref 0x0~2000 Blob(0x558e84190310 blob([0x86000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf730 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e84190310) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e84190310 blob([0x86000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edf730 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xea858 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83edf748 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x858 2019-12-07T19:34:08.621+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.621+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xccd and padding block with 0x333 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84549e30 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84549ea0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84549e30 blob([0x88000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84549ea0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e84549e30) get_ref 0x0~2000 Blob(0x558e84549e30 blob([0x88000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84549ea0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e84549e30) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e84549e30 blob([0x88000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84549ea0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e84549eb8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84549f10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84549f80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84549f10 blob([0x8a000~1000,0x1c3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84549f80 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e84549f10) get_ref 0x0~2000 Blob(0x558e84549f10 blob([0x8a000~1000,0x1c3000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84549f80 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e84549f10) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e84549f10 blob([0x8a000~1000,0x1c3000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84549f80 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e84549f98 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d027e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d02930 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d027e0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02930 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83d027e0) get_ref 0x0~2000 Blob(0x558e83d027e0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02930 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83d027e0) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e83d027e0 blob([0x1c4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02930 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83d02948 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d02690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d02690 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4a10 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83d02690) get_ref 0x0~2000 Blob(0x558e83d02690 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac4a10 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83d02690) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e83d02690 blob([0x1c6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac4a10 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83ac4a28 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ac4d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ac4d90 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5490 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83ac4d90) get_ref 0x0~2000 Blob(0x558e83ac4d90 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83ac5490 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83ac4d90) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83ac4d90 blob([0x1c8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83ac5490 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83ac54a8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d36fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d36fc0 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fc70 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83d36fc0) get_ref 0x0~2000 Blob(0x558e83d36fc0 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6fc70 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83d36fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e83d36fc0 blob([0x1ca000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6fc70 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83e6fc88 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd2fc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2c40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd2fc0 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2c40 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83dd2fc0) get_ref 0x0~2000 Blob(0x558e83dd2fc0 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2c40 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83dd2fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e83dd2fc0 blob([0x1cc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2c40 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83dd2c58 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd2b60 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2af0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd2b60 blob([0x1ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2af0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83dd2b60) get_ref 0x0~2000 Blob(0x558e83dd2b60 blob([0x1ce000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2af0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83dd2b60) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e83dd2b60 blob([0x1ce000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2af0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83dd2b08 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd29a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2380 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd29a0 blob([0x1d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2380 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83dd29a0) get_ref 0x0~2000 Blob(0x558e83dd29a0 blob([0x1d0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2380 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83dd29a0) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e83dd29a0 blob([0x1d0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2380 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83dd2398 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84030f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83b288c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84030f50 blob([0x1d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b288c0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e84030f50) get_ref 0x0~2000 Blob(0x558e84030f50 blob([0x1d2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83b288c0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e84030f50) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e84030f50 blob([0x1d2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83b288c0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e83b288d8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83b28380 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841a2460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83b28380 blob([0x1d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a2460 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83b28380) get_ref 0x0~2000 Blob(0x558e83b28380 blob([0x1d4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a2460 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e83b28380) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e83b28380 blob([0x1d4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2460 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e841a2478 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c4150 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846c41c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11accd 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c4150 blob([0x1d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c41c0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c4150) get_ref 0x0~2000 Blob(0x558e846c4150 blob([0x1d6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c41c0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c4150) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e846c4150 blob([0x1d6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846c41c0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e846c41d8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.621+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c4230 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846c42a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.621+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c4230 blob([0x1d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c42a0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c4230) get_ref 0x0~2000 Blob(0x558e846c4230 blob([0x1d8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c42a0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c4230) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e846c4230 blob([0x1d8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846c42a0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e846c42b8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c4310 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846c50a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000233464s (0.000000240s flush + 0.000233224s kv commit) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c4310 blob([0x1da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c50a0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c4310) get_ref 0x0~2000 Blob(0x558e846c4310 blob([0x1da000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c50a0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c4310) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e846c4310 blob([0x1da000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846c50a0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4c300> 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e846c50b8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.621+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 kv_submitted 2019-12-07T19:34:08.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c300 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c5110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846c53b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 finishing 2019-12-07T19:34:08.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c300 onodes 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c5110 blob([0x1dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c53b0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c300 done 2019-12-07T19:34:08.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:08.621+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c300 [] 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c5110) get_ref 0x0~2000 Blob(0x558e846c5110 blob([0x1dc000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c53b0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c5110) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e846c5110 blob([0x1dc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846c53b0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e846c53c8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c5180 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846c52d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c5180 blob([0x1de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c52d0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c5180) get_ref 0x0~2000 Blob(0x558e846c5180 blob([0x1de000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c52d0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c5180) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558e846c5180 blob([0x1de000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846c52d0 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e846c52e8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c5340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846c5420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c5340 blob([0x1e0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c5420 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c5340) get_ref 0x0~2000 Blob(0x558e846c5340 blob([0x1e0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c5420 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c5340) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558e846c5340 blob([0x1e0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846c5420 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e846c5438 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c55e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846c5570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c55e0 blob([0x1e2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c5570 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c55e0) get_ref 0x0~2000 Blob(0x558e846c55e0 blob([0x1e2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846c5570 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.blob(0x558e846c55e0) get_ref init 0x2000, 1000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558e846c55e0 blob([0x1e2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846c5570 sbid 0x0)) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.BufferSpace(0x558e846c5588 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x30000 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e83c73508) dirty_range mark inline shard dirty 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 0x0~30000 = 0 2019-12-07T19:34:08.621+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# hinfo_key (42 bytes) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:08.621+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000031.00000000000000000030' <- 0000000031.00000000000000000030 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:08.621+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e73500 cost 303020 (26 ios * 4000 + 199020 bytes) 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e73500 onodes 0x558e83c73400 shared_blobs 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e83c73508) update 5#1:14fc029d:::smithi13140796-41:head# 2019-12-07T19:34:08.621+0000 7fe2cba6e700 20 bluestore.extentmap(0x558e83c73508) update inline shard 464 bytes from 24 extents 2019-12-07T19:34:08.622+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:14fc029d:::smithi13140796-41:head# is 883 (413 bytes onode + 2 bytes spanning blobs + 468 bytes inline extents) 2019-12-07T19:34:08.622+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e73500 allocated 0x[7c000~f000,1c3000~21000] released 0x[] 2019-12-07T19:34:08.622+0000 7fe2cba6e700 10 freelist allocate 0x7c000~f000 2019-12-07T19:34:08.622+0000 7fe2cba6e700 20 freelist _xor first_key 0x0 last_key 0x80000 2019-12-07T19:34:08.622+0000 7fe2cba6e700 10 freelist allocate 0x1c3000~21000 2019-12-07T19:34:08.622+0000 7fe2cba6e700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:34:08.622+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73500 prepare 2019-12-07T19:34:08.622+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83e73500 2019-12-07T19:34:08.622+0000 7fe2cba6e700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'30 (0'0,31'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=29'29 crt=29'29 lcod 29'29 active mbc={} ps=[4~1]] op_applied version 31'30 2019-12-07T19:34:08.622+0000 7fe2cba6e700 10 osd.0 31 dequeue_op 0x558e8477b340 finish 2019-12-07T19:34:08.622+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.622+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73500 aio_wait 2019-12-07T19:34:08.622+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e73500 2019-12-07T19:34:08.622+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73500 io_done 2019-12-07T19:34:08.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:08.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2443 < min_flush_size 524288 2019-12-07T19:34:08.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c73400 had 1 2019-12-07T19:34:08.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:08.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2466 < min_flush_size 524288 2019-12-07T19:34:08.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2466 < min_flush_size 524288 2019-12-07T19:34:08.622+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.622+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11accd~9a2 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.622+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11accd 0x11accd~9a2 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.622+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.622+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xeaccd 2019-12-07T19:34:08.622+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xccd 2019-12-07T19:34:08.622+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.622+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x66f and padding block with 0x991 2019-12-07T19:34:08.622+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11b66f 2019-12-07T19:34:08.622+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.622+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.622+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.622+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000120757s (0.000005095s flush + 0.000115662s kv commit) 2019-12-07T19:34:08.622+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e73500> 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73500 kv_submitted 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e73500 2019-12-07T19:34:08.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73500 finishing 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e73500 onodes 0x558e83c73400 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac4a28 in 0x558e80219960) _finish_write discard buffer(0x558e843d35f0 space 0x558e83ac4a28 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac4e18 in 0x558e80219960) _finish_write discard buffer(0x558e84597cb0 space 0x558e83ac4e18 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83ac54a8 in 0x558e80219960) _finish_write discard buffer(0x558e843d3680 space 0x558e83ac54a8 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83b288d8 in 0x558e80219960) _finish_write discard buffer(0x558e843d3b00 space 0x558e83b288d8 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d02948 in 0x558e80219960) _finish_write discard buffer(0x558e843d3c20 space 0x558e83d02948 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2398 in 0x558e80219960) _finish_write discard buffer(0x558e843d25a0 space 0x558e83dd2398 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2b08 in 0x558e80219960) _finish_write discard buffer(0x558e843d3830 space 0x558e83dd2b08 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd2c58 in 0x558e80219960) _finish_write discard buffer(0x558e843d37a0 space 0x558e83dd2c58 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6fc88 in 0x558e80219960) _finish_write discard buffer(0x558e843d3710 space 0x558e83e6fc88 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edf748 in 0x558e80219960) _finish_write discard buffer(0x558e843d33b0 space 0x558e83edf748 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84190868 in 0x558e80219960) _finish_write discard buffer(0x558e843d2510 space 0x558e84190868 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841a2478 in 0x558e80219960) _finish_write discard buffer(0x558e843d38c0 space 0x558e841a2478 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84549eb8 in 0x558e80219960) _finish_write discard buffer(0x558e843d3a70 space 0x558e84549eb8 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84549f98 in 0x558e80219960) _finish_write discard buffer(0x558e843d3b90 space 0x558e84549f98 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846c4088 in 0x558e80219960) _finish_write discard buffer(0x558e84597e60 space 0x558e846c4088 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846c41d8 in 0x558e80219960) _finish_write discard buffer(0x558e843d3950 space 0x558e846c41d8 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846c42b8 in 0x558e80219960) _finish_write discard buffer(0x558e843d39e0 space 0x558e846c42b8 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846c50b8 in 0x558e80219960) _finish_write discard buffer(0x558e843d3ef0 space 0x558e846c50b8 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846c5278 in 0x558e80219960) _finish_write discard buffer(0x558e84597dd0 space 0x558e846c5278 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846c52e8 in 0x558e80219960) _finish_write discard buffer(0x558e843d3dd0 space 0x558e846c52e8 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846c53c8 in 0x558e80219960) _finish_write discard buffer(0x558e843d3e60 space 0x558e846c53c8 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846c5438 in 0x558e80219960) _finish_write discard buffer(0x558e843d3d40 space 0x558e846c5438 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846c5588 in 0x558e80219960) _finish_write discard buffer(0x558e843d3cb0 space 0x558e846c5588 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846c56d8 in 0x558e80219960) _finish_write discard buffer(0x558e84597d40 space 0x558e846c56d8 0x0~2000 writing nocache) 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e73500 done 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:08.622+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e73500 [] 2019-12-07T19:34:08.622+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.622+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.622+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.622+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 31/28 ECSubWriteReply(tid=149, last_complete=31'30, committed=1, applied=1)) v2 -- 0x558e84053000 con 0x558e818d7c00 2019-12-07T19:34:08.622+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.623+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 527028 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 233 ==== MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=150, reqid=client.4540.0:262, at_version=31'31, trim_to=0'0, roll_forward_to=29'29)) v2 ==== 96536+0+0 (unknown 0 0 0) 0x558e83aca800 con 0x558e818d7c00 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 15 osd.0 31 enqueue_op 0x558e8477b4a0 prio 127 cost 0 latency 0.000034 epoch 31 MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=150, reqid=client.4540.0:262, at_version=31'31, trim_to=0'0, roll_forward_to=29'29)) v2 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=150, reqid=client.4540.0:262, at_version=31'31, trim_to=0'0, roll_forward_to=29'29)) v2) prio 127 cost 0 e31) 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 234 ==== MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=151, reqid=client.4540.0:263, at_version=31'32, trim_to=0'0, roll_forward_to=29'29)) v2 ==== 2234+0+0 (unknown 0 0 0) 0x558e84043800 con 0x558e818d7c00 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 15 osd.0 31 enqueue_op 0x558e8477b600 prio 127 cost 0 latency 0.000018 epoch 31 MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=151, reqid=client.4540.0:263, at_version=31'32, trim_to=0'0, roll_forward_to=29'29)) v2 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=151, reqid=client.4540.0:263, at_version=31'32, trim_to=0'0, roll_forward_to=29'29)) v2) prio 127 cost 0 e31) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=150, reqid=client.4540.0:262, at_version=31'31, trim_to=0'0, roll_forward_to=29'29)) v2) prio 127 cost 0 e31) queued 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=150, reqid=client.4540.0:262, at_version=31'31, trim_to=0'0, roll_forward_to=29'29)) v2) prio 127 cost 0 e31) pg 0x558e841a4000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 31 dequeue_op 0x558e8477b4a0 prio 127 cost 0 latency 0.000141 MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=150, reqid=client.4540.0:262, at_version=31'31, trim_to=0'0, roll_forward_to=29'29)) v2 pg pg[1.8s5( v 31'30 (0'0,31'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[4~1]] 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'30 (0'0,31'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[4~1]] _handle_message: MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=150, reqid=client.4540.0:262, at_version=31'31, trim_to=0'0, roll_forward_to=29'29)) v2 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'30 (0'0,31'30] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[4~1]] append_log log((0'0,31'30], crt=29'29) [31'31 (31'30) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:262 2019-12-07T19:34:08.615254+0000 0 ObjectCleanRegions clean_offsets: [0~786432,1163264~18446744073708388351], clean_omap: 1, new_object: 0] 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'31 (0'0,31'31] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=31'30 crt=29'29 lcod 31'30 active mbc={} ps=[4~1]] add_log_entry 31'31 (31'30) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:262 2019-12-07T19:34:08.615254+0000 0 ObjectCleanRegions clean_offsets: [0~786432,1163264~18446744073708388351], clean_omap: 1, new_object: 0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'31 (0'0,31'31] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=31'30 crt=29'29 lcod 31'30 active mbc={} ps=[4~1]] append_log approx pg log length = 31 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'31 (0'0,31'31] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=31'30 crt=29'29 lcod 31'30 active mbc={} ps=[4~1]] append_log transaction_applied = 1 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 31'31, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e811fa300 seq 35 2019-12-07T19:34:08.624+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 2 keys 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 2 keys = 0 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 124 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 0x30000~17000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#1:14fc029d:::smithi13140796-41:head# 0x30000~17000 - have 0x30000 (196608) bytes fadvise_flags 0x20 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x30000~17000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.1s1 epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e811fa300 12 blobs 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 125 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.1s1 epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 126 ==== pg_info2(1.fs1 1.fs1 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.fs1 epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.624+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate 0x17000/1000,17000,0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x1e4000~6000/1000,17000,0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x2a4000~11000/1000,17000,0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x1e4000~6000,0x2a4000~11000] 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84645340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846453b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84645340 blob([0x1e4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846453b0 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645340) get_ref 0x0~2000 Blob(0x558e84645340 blob([0x1e4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846453b0 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e84645340) get_ref init 0x2000, 1000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558e84645340 blob([0x1e4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846453b0 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e846453c8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.624+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.624+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.624+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84548cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f61730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.624+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e31) pg 0x558e81a9b000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84548cb0 blob([0x1e6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61730 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548cb0) get_ref 0x0~2000 Blob(0x558e84548cb0 blob([0x1e6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61730 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558e84548cb0 blob([0x1e6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f61730 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f61748 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.624+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f617a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f617a0 blob([0x1e8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645260 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f617a0) get_ref 0x0~2000 Blob(0x558e83f617a0 blob([0x1e8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645260 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f617a0) get_ref init 0x2000, 1000 2019-12-07T19:34:08.624+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558e83f617a0 blob([0x1e8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645260 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645278 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846452d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846452d0 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645810 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e846452d0) get_ref 0x0~2000 Blob(0x558e846452d0 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645810 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e846452d0) get_ref init 0x2000, 1000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558e846452d0 blob([0x2a4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645810 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645828 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a7e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f61420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e811fb500 seq 35 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a7e0 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61420 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a7e0) get_ref 0x0~2000 Blob(0x558e8456a7e0 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61420 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a7e0) get_ref init 0x2000, 1000 2019-12-07T19:34:08.624+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558e8456a7e0 blob([0x2a6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f61420 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f61438 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f610a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f610a0 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645420 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f610a0) get_ref 0x0~2000 Blob(0x558e83f610a0 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645420 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f610a0) get_ref init 0x2000, 1000 2019-12-07T19:34:08.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fb500 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558e83f610a0 blob([0x2a8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645420 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fb500 onodes shared_blobs 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84645438 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fb500 allocated 0x[] released 0x[] 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f619d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f619d0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b420 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f619d0) get_ref 0x0~2000 Blob(0x558e83f619d0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b420 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 prepare 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e83f619d0) get_ref init 0x2000, 1000 2019-12-07T19:34:08.624+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fb500 2019-12-07T19:34:08.624+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 io_done 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558e83f619d0 blob([0x2aa000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5b420 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e83f5b438 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84548d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456a770 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.624+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84548d90 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a770 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548d90) get_ref 0x0~2000 Blob(0x558e84548d90 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a770 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e84548d90) get_ref init 0x2000, 1000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558e84548d90 blob([0x2ac000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a770 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a788 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456ae00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84549650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456ae00 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84549650 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456ae00) get_ref 0x0~2000 Blob(0x558e8456ae00 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84549650 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456ae00) get_ref init 0x2000, 1000 2019-12-07T19:34:08.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558e8456ae00 blob([0x2ae000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84549650 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e84549668 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a4d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456a540 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a4d0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a540 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a4d0) get_ref 0x0~2000 Blob(0x558e8456a4d0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a540 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a4d0) get_ref init 0x2000, 1000 2019-12-07T19:34:08.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1141 < min_flush_size 524288 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558e8456a4d0 blob([0x2b0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a540 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a558 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.624+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456ad90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e845495e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:08.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11b66f~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456ad90 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e845495e0 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11b66f 0x11b66f~475 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456ad90) get_ref 0x0~2000 Blob(0x558e8456ad90 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e845495e0 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456ad90) get_ref init 0x2000, 1000 2019-12-07T19:34:08.624+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558e8456ad90 blob([0x2b2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e845495e0 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xeb66f 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e845495f8 in 0x558e80219960) _discard 0x0~2000 2019-12-07T19:34:08.624+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x66f 2019-12-07T19:34:08.624+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.624+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xae4 and padding block with 0x51c 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a3f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456a460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a3f0 blob([0x2b4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a460 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a3f0) get_ref 0x0~1000 Blob(0x558e8456a3f0 blob([0x2b4000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a460 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.blob(0x558e8456a3f0) get_ref init 0x1000, 1000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x46000~1000: 0x0~1000 Blob(0x558e8456a3f0 blob([0x2b4000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8456a460 sbid 0x0)) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.BufferSpace(0x558e8456a478 in 0x558e80219960) _discard 0x0~1000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x47000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e83c73508) dirty_range mark inline shard dirty 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 0x30000~17000 = 0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# hinfo_key (42 bytes) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000031.00000000000000000031' <- 0000000031.00000000000000000031 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._fastinfo' <- _fastinfo 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa300 cost 148186 (13 ios * 4000 + 96186 bytes) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa300 onodes 0x558e83c73400 shared_blobs 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e83c73508) update 5#1:14fc029d:::smithi13140796-41:head# 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e83c73508) update inline shard 689 bytes from 36 extents 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:14fc029d:::smithi13140796-41:head# is 1108 (413 bytes onode + 2 bytes spanning blobs + 693 bytes inline extents) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa300 allocated 0x[1e4000~6000,2a4000~11000] released 0x[] 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 freelist allocate 0x1e4000~6000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 freelist _xor first_key 0x180000 last_key 0x180000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 freelist allocate 0x2a4000~11000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 prepare 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e811fa300 2019-12-07T19:34:08.624+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11bae4 2019-12-07T19:34:08.624+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.624+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.624+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.624+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000171330s (0.000000093s flush + 0.000171237s kv commit) 2019-12-07T19:34:08.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fb500> 2019-12-07T19:34:08.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.624+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 kv_submitted 2019-12-07T19:34:08.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fb500 2019-12-07T19:34:08.624+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fb500 finishing 2019-12-07T19:34:08.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fb500 onodes 2019-12-07T19:34:08.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fb500 done 2019-12-07T19:34:08.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:08.624+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fb500 [] 2019-12-07T19:34:08.624+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.624+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.624+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'31 (0'0,31'31] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=31'30 crt=29'29 lcod 31'30 active mbc={} ps=[4~1]] op_applied version 31'31 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 31 dequeue_op 0x558e8477b4a0 finish 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=151, reqid=client.4540.0:263, at_version=31'32, trim_to=0'0, roll_forward_to=29'29)) v2) prio 127 cost 0 e31) queued 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=151, reqid=client.4540.0:263, at_version=31'32, trim_to=0'0, roll_forward_to=29'29)) v2) prio 127 cost 0 e31) pg 0x558e841a4000 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 31 dequeue_op 0x558e8477b600 prio 127 cost 0 latency 0.000694 MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=151, reqid=client.4540.0:263, at_version=31'32, trim_to=0'0, roll_forward_to=29'29)) v2 pg pg[1.8s5( v 31'31 (0'0,31'31] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 lcod 31'30 active mbc={} ps=[4~1]] 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'31 (0'0,31'31] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 lcod 31'30 active mbc={} ps=[4~1]] _handle_message: MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=151, reqid=client.4540.0:263, at_version=31'32, trim_to=0'0, roll_forward_to=29'29)) v2 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'31 (0'0,31'31] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 lcod 31'30 active mbc={} ps=[4~1]] append_log log((0'0,31'31], crt=29'29) [31'32 (31'31) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:263 2019-12-07T19:34:08.615270+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=31'31 crt=29'29 lcod 31'30 active mbc={} ps=[4~1]] add_log_entry 31'32 (31'31) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:263 2019-12-07T19:34:08.615270+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=31'31 crt=29'29 lcod 31'30 active mbc={} ps=[4~1]] append_log approx pg log length = 32 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=31'31 crt=29'29 lcod 31'30 active mbc={} ps=[4~1]] append_log transaction_applied = 1 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 31'32, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e81a0b200 seq 36 2019-12-07T19:34:08.624+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 3 keys 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# 3 keys = 0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# hinfo_key (42 bytes) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.8s5_head 5#1:14fc029d:::smithi13140796-41:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000031.00000000000000000032' <- 0000000031.00000000000000000032 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._fastinfo' <- _fastinfo 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b200 cost 5884 (1 ios * 4000 + 1884 bytes) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b200 onodes 0x558e83c73400 shared_blobs 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e83c73508) update 5#1:14fc029d:::smithi13140796-41:head# 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:14fc029d:::smithi13140796-41:head# is 1180 (485 bytes onode + 2 bytes spanning blobs + 693 bytes inline extents) 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b200 allocated 0x[] released 0x[] 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b200 prepare 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b200 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b200 blocked by 0x558e811fa300 aio_wait 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=31'31 crt=29'29 lcod 31'30 active mbc={} ps=[4~1]] op_applied version 31'32 2019-12-07T19:34:08.624+0000 7fe2c7a66700 10 osd.0 31 dequeue_op 0x558e8477b600 finish 2019-12-07T19:34:08.624+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.624+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 aio_wait 2019-12-07T19:34:08.624+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa300 2019-12-07T19:34:08.624+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 io_done 2019-12-07T19:34:08.624+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b200 io_done 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e31) pg 0x558e819e4000 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[2~1,4~1]] proc_primary_info updating purged_snaps to [4~1] 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e819a0600 seq 29 2019-12-07T19:34:08.625+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0600 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0600 onodes shared_blobs 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0600 allocated 0x[] released 0x[] 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0600 prepare 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0600 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0600 io_done 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e31) pg 0x558e819e4000 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e83e73200 seq 30 2019-12-07T19:34:08.625+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e73200 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e73200 onodes shared_blobs 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e73200 allocated 0x[] released 0x[] 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73200 prepare 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e73200 2019-12-07T19:34:08.625+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73200 io_done 2019-12-07T19:34:08.625+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 235 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.8s5 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 236 ==== pg_info2(1.as3 1.as3 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.as3 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 237 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.es4 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.626+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.626+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 238 ==== pg_info2(1.7s3 1.7s3 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:08.626+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.626+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e31) pg 0x558e81a98000 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.7s3 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.626+0000 7fe2c8a68700 10 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 239 ==== pg_info2(1.9s3 1.9s3 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.9s3 epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:08.626+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.626+0000 7fe2c8a68700 10 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.626+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:08.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.626+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:08.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.626+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e811fa600 seq 26 2019-12-07T19:34:08.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.626+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e31) pg 0x558e8192b000 2019-12-07T19:34:08.626+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.626+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.626+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e31) pg 0x558e8193b000 2019-12-07T19:34:08.626+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:08.626+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:08.626+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:08.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:08.626+0000 7fe2c6a64700 10 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:08.626+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:08.626+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fa600 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.626+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fa600 onodes shared_blobs 2019-12-07T19:34:08.626+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fa600 allocated 0x[] released 0x[] 2019-12-07T19:34:08.626+0000 7fe2c7265700 10 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.626+0000 7fe2c6a64700 10 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.626+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 prepare 2019-12-07T19:34:08.626+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fa600 2019-12-07T19:34:08.626+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.626+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 io_done 2019-12-07T19:34:08.626+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.626+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:08.626+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.626+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:08.626+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e31) pg 0x558e8122e000 2019-12-07T19:34:08.626+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:08.626+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:08.626+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e8195e300 seq 22 2019-12-07T19:34:08.626+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e8195f800 seq 33 2019-12-07T19:34:08.626+0000 7fe2c8267700 10 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:08.626+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:08.626+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:08.626+0000 7fe2c8267700 10 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.626+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:08.626+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:08.626+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:08.626+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:08.626+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:08.626+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:08.626+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:08.626+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e300 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.626+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195f800 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.626+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e300 onodes shared_blobs 2019-12-07T19:34:08.626+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195f800 onodes shared_blobs 2019-12-07T19:34:08.626+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e300 allocated 0x[] released 0x[] 2019-12-07T19:34:08.626+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195f800 allocated 0x[] released 0x[] 2019-12-07T19:34:08.626+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:08.626+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e81a0a300 seq 36 2019-12-07T19:34:08.626+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 prepare 2019-12-07T19:34:08.626+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e300 2019-12-07T19:34:08.626+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 io_done 2019-12-07T19:34:08.626+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:08.626+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:08.626+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:08.626+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:08.626+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0a300 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.626+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0a300 onodes shared_blobs 2019-12-07T19:34:08.626+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0a300 allocated 0x[] released 0x[] 2019-12-07T19:34:08.626+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 prepare 2019-12-07T19:34:08.626+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0a300 2019-12-07T19:34:08.626+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 io_done 2019-12-07T19:34:08.626+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 7 submitting 7 deferred done 0 stable 0 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:08.626+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 prepare 2019-12-07T19:34:08.626+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195f800 2019-12-07T19:34:08.626+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 io_done 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2277 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c73400 had 2 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 2 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4497 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c73400 had 1 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5615 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 2 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 6725 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 7835 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8945 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 10055 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 10078 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 10078 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11bae4~275e to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11bae4 0x11bae4~275e to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xebae4 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xae4 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x242 and padding block with 0xdbe 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11e242 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 7 cleaned 0 in 0.000223883s (0.000003794s flush + 0.000220089s kv commit) 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11e242~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11e242 0x11e242~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xee242 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x242 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x6af and padding block with 0x951 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fa300,0x558e81a0b200,0x558e819a0600,0x558e83e73200,0x558e811fa600,0x558e8195e300,0x558e81a0a300> 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 kv_submitted 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa300 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa300 finishing 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa300 onodes 0x558e83c73400 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5b438 in 0x558e80219960) _finish_write discard buffer(0x558e842b9e60 space 0x558e83f5b438 0x0~2000 writing nocache) 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f61438 in 0x558e80219960) _finish_write discard buffer(0x558e842b9cb0 space 0x558e83f61438 0x0~2000 writing nocache) 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f61748 in 0x558e80219960) _finish_write discard buffer(0x558e842b8bd0 space 0x558e83f61748 0x0~2000 writing nocache) 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e845495f8 in 0x558e80219960) _finish_write discard buffer(0x558e84758c60 space 0x558e845495f8 0x0~2000 writing nocache) 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84549668 in 0x558e80219960) _finish_write discard buffer(0x558e84758000 space 0x558e84549668 0x0~2000 writing nocache) 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456a478 in 0x558e80219960) _finish_write discard buffer(0x558e84758cf0 space 0x558e8456a478 0x0~1000 writing nocache) 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456a558 in 0x558e80219960) _finish_write discard buffer(0x558e84758bd0 space 0x558e8456a558 0x0~2000 writing nocache) 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456a788 in 0x558e80219960) _finish_write discard buffer(0x558e842b9ef0 space 0x558e8456a788 0x0~2000 writing nocache) 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645278 in 0x558e80219960) _finish_write discard buffer(0x558e842b9c20 space 0x558e84645278 0x0~2000 writing nocache) 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846453c8 in 0x558e80219960) _finish_write discard buffer(0x558e842b95f0 space 0x558e846453c8 0x0~2000 writing nocache) 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645438 in 0x558e80219960) _finish_write discard buffer(0x558e842b9dd0 space 0x558e84645438 0x0~2000 writing nocache) 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645828 in 0x558e80219960) _finish_write discard buffer(0x558e84032480 space 0x558e84645828 0x0~2000 writing nocache) 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa300 done 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b200 kv_submitted 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa300 [] 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b200 kv_submitted 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b200 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b200 finishing 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b200 onodes 0x558e83c73400 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b200 done 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b200 [] 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0600 kv_submitted 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0600 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0600 finishing 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0600 onodes 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0600 done 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e73200 kv_submitted 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0600 [] 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73200 kv_submitted 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e73200 2019-12-07T19:34:08.626+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73200 finishing 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e73200 onodes 2019-12-07T19:34:08.626+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e73200 done 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:08.626+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e73200 [] 2019-12-07T19:34:08.626+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e31) pg 0x558e841a4000 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 kv_submitted 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fa600 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa600 finishing 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa600 onodes 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa600 done 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa600 [] 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 kv_submitted 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e300 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 finishing 2019-12-07T19:34:08.626+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e300 onodes 2019-12-07T19:34:08.626+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 lcod 31'30 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 done 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e300 [] 2019-12-07T19:34:08.626+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11e6af 2019-12-07T19:34:08.626+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.626+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 lcod 31'30 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000147992s (0.000000070s flush + 0.000147922s kv commit) 2019-12-07T19:34:08.626+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 kv_submitted 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0a300 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0a300 finishing 2019-12-07T19:34:08.626+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0a300 onodes 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0a300 done 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:08.626+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e81a0b800 seq 37 2019-12-07T19:34:08.626+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0a300 [] 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.626+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195f800> 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 kv_submitted 2019-12-07T19:34:08.626+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195f800 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 finishing 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195f800 onodes 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f800 done 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:08.626+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._biginfo' <- _biginfo 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195f800 [] 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.626+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:08.626+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:08.626+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b800 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.626+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b800 onodes shared_blobs 2019-12-07T19:34:08.626+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b800 allocated 0x[] released 0x[] 2019-12-07T19:34:08.626+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 prepare 2019-12-07T19:34:08.626+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b800 2019-12-07T19:34:08.626+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 io_done 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.626+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11e6af~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11e6af 0x11e6af~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xee6af 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x6af 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb1c and padding block with 0x4e4 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11eb1c 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.626+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000118800s (0.000000094s flush + 0.000118706s kv commit) 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.626+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0b800> 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 kv_submitted 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b800 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b800 finishing 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b800 onodes 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b800 done 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b800 [] 2019-12-07T19:34:08.626+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.626+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.627+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.627+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 127 ==== pg_info2(1.fs1 1.fs1 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e31/31) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7000 2019-12-07T19:34:08.627+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.fs1 epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:08.627+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e31) 2019-12-07T19:34:08.627+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.627+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e31) queued 2019-12-07T19:34:08.627+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.627+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e31) pg 0x558e81a9b000 2019-12-07T19:34:08.627+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1]] do_peering_event: epoch_sent: 31 epoch_requested: 31 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:08.627+0000 7fe2c425f700 10 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1]] proc_primary_info updating purged_snaps to [4~1,7~1] 2019-12-07T19:34:08.627+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:08.627+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:08.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e811faf00 seq 36 2019-12-07T19:34:08.627+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:08.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:08.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:08.627+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:08.627+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811faf00 cost 5273 (1 ios * 4000 + 1273 bytes) 2019-12-07T19:34:08.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811faf00 onodes shared_blobs 2019-12-07T19:34:08.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811faf00 allocated 0x[] released 0x[] 2019-12-07T19:34:08.627+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 prepare 2019-12-07T19:34:08.627+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811faf00 2019-12-07T19:34:08.627+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 io_done 2019-12-07T19:34:08.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.627+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1110 < min_flush_size 524288 2019-12-07T19:34:08.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:08.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1133 < min_flush_size 524288 2019-12-07T19:34:08.627+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.627+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11eb1c~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.627+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11eb1c 0x11eb1c~46d to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.627+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.627+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xeeb1c 2019-12-07T19:34:08.627+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb1c 2019-12-07T19:34:08.627+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.627+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 31/28 ECSubWriteReply(tid=150, last_complete=31'31, committed=1, applied=1)) v2 -- 0x558e819df600 con 0x558e818d7c00 2019-12-07T19:34:08.627+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xf89 and padding block with 0x77 2019-12-07T19:34:08.627+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 31/28 ECSubWriteReply(tid=151, last_complete=31'32, committed=1, applied=1)) v2 -- 0x558e81954000 con 0x558e818d7c00 2019-12-07T19:34:08.627+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.627+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11ef89 2019-12-07T19:34:08.627+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.627+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.627+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.627+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000138928s (0.000000081s flush + 0.000138847s kv commit) 2019-12-07T19:34:08.627+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811faf00> 2019-12-07T19:34:08.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.627+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 kv_submitted 2019-12-07T19:34:08.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811faf00 2019-12-07T19:34:08.627+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811faf00 finishing 2019-12-07T19:34:08.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811faf00 onodes 2019-12-07T19:34:08.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811faf00 done 2019-12-07T19:34:08.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:08.627+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811faf00 [] 2019-12-07T19:34:08.627+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.627+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.627+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.627+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:08.629+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 240 ==== MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=152, reqid=client.4540.0:263, at_version=0'0, trim_to=0'0, roll_forward_to=31'32)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83fec800 con 0x558e818d7c00 2019-12-07T19:34:08.629+0000 7fe2e7d3c700 15 osd.0 31 enqueue_op 0x558e8477b760 prio 127 cost 0 latency 0.000024 epoch 31 MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=152, reqid=client.4540.0:263, at_version=0'0, trim_to=0'0, roll_forward_to=31'32)) v2 2019-12-07T19:34:08.629+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=152, reqid=client.4540.0:263, at_version=0'0, trim_to=0'0, roll_forward_to=31'32)) v2) prio 127 cost 0 e31) 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=152, reqid=client.4540.0:263, at_version=0'0, trim_to=0'0, roll_forward_to=31'32)) v2) prio 127 cost 0 e31) queued 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=152, reqid=client.4540.0:263, at_version=0'0, trim_to=0'0, roll_forward_to=31'32)) v2) prio 127 cost 0 e31) pg 0x558e841a4000 2019-12-07T19:34:08.629+0000 7fe2c7a66700 10 osd.0 31 dequeue_op 0x558e8477b760 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=152, reqid=client.4540.0:263, at_version=0'0, trim_to=0'0, roll_forward_to=31'32)) v2 pg pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[4~1,7~1]] 2019-12-07T19:34:08.629+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[4~1,7~1]] _handle_message: MOSDECSubOpWrite(1.8s5 31/28 ECSubWrite(tid=152, reqid=client.4540.0:263, at_version=0'0, trim_to=0'0, roll_forward_to=31'32)) v2 2019-12-07T19:34:08.629+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=29'29 active mbc={} ps=[4~1,7~1]] append_log log((0'0,31'32], crt=29'29) [] 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] rollforward: entry=31'30 (0'0) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:261 2019-12-07T19:34:08.612951+0000 0 ObjectCleanRegions clean_offsets: [786432~18446744073708765183], clean_omap: 1, new_object: 0 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] rollforward: entry=31'31 (31'30) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:262 2019-12-07T19:34:08.615254+0000 0 ObjectCleanRegions clean_offsets: [0~786432,1163264~18446744073708388351], clean_omap: 1, new_object: 0 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] rollforward: entry=31'32 (31'31) modify 1:14fc029d:::smithi13140796-41:head by client.4540.0:263 2019-12-07T19:34:08.615270+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:08.629+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] append_log approx pg log length = 32 2019-12-07T19:34:08.629+0000 7fe2c7a66700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] append_log transaction_applied = 1 2019-12-07T19:34:08.629+0000 7fe2c7a66700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:08.629+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:08.629+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e84559500 seq 38 2019-12-07T19:34:08.629+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:08.629+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:08.629+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84559500 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84559500 onodes shared_blobs 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84559500 allocated 0x[] released 0x[] 2019-12-07T19:34:08.629+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559500 prepare 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84559500 2019-12-07T19:34:08.629+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559500 io_done 2019-12-07T19:34:08.629+0000 7fe2c7a66700 10 osd.0 31 dequeue_op 0x558e8477b760 finish 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:08.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:08.629+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:08.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:08.629+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:08.629+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:08.629+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.629+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11ef89~471 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.629+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11ef89 0x11ef89~471 to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.629+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:08.629+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~f0000 x_off 0xeef89 2019-12-07T19:34:08.629+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xf89 2019-12-07T19:34:08.629+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:08.629+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x3fa and padding block with 0xc06 2019-12-07T19:34:08.629+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x11f3fa 2019-12-07T19:34:08.629+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:08.629+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:08.629+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:08.629+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:08.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000101979s (0.000000085s flush + 0.000101894s kv commit) 2019-12-07T19:34:08.629+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:08.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:08.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e84559500> 2019-12-07T19:34:08.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:08.629+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559500 kv_submitted 2019-12-07T19:34:08.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84559500 2019-12-07T19:34:08.629+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559500 finishing 2019-12-07T19:34:08.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84559500 onodes 2019-12-07T19:34:08.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84559500 done 2019-12-07T19:34:08.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:08.629+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84559500 [] 2019-12-07T19:34:08.629+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:08.629+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:08.629+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.629+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 31/28 ECSubWriteReply(tid=152, last_complete=31'32, committed=1, applied=1)) v2 -- 0x558e83e68800 con 0x558e818d7c00 2019-12-07T19:34:08.629+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:08.653+0000 7fe2c3a5e700 5 osd.0 31 heartbeat osd_stat(store_statfs(0x163fb1d000/0x40000000/0x1680000000, data 0x4afe87/0x4e1000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5,7] op hist []) 2019-12-07T19:34:08.654+0000 7fe2c3a5e700 20 osd.0 31 check_full_status cur ratio 5.30243e-05, physical ratio 5.30243e-05, new state none 2019-12-07T19:34:08.654+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.109405351s delta_ub 18446744073.707482502s) v5 -- 0x558e83e68a00 con 0x558e818d7800 2019-12-07T19:34:08.654+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.109405351s delta_ub 18446744073.707482502s) v5 -- 0x558e83e68c00 con 0x558e818d7400 2019-12-07T19:34:08.654+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.109405351s delta_ub 18446744073.692616958s) v5 -- 0x558e83e68e00 con 0x558e817ff000 2019-12-07T19:34:08.654+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.109405351s delta_ub 18446744073.692616958s) v5 -- 0x558e83e69000 con 0x558e817ff400 2019-12-07T19:34:08.654+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.109405351s delta_ub 18446744073.706876433s) v5 -- 0x558e83e69200 con 0x558e817ff800 2019-12-07T19:34:08.654+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.109405351s delta_ub 18446744073.706876433s) v5 -- 0x558e83e69400 con 0x558e817ffc00 2019-12-07T19:34:08.654+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.109405351s delta_ub 0.005677330s) v5 -- 0x558e83e69600 con 0x558e8181a000 2019-12-07T19:34:08.654+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.109405351s delta_ub 0.005677330s) v5 -- 0x558e83e69800 con 0x558e8181a400 2019-12-07T19:34:08.654+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.109405351s delta_ub 18446744073.698237236s) v5 -- 0x558e83e69a00 con 0x558e817fe000 2019-12-07T19:34:08.654+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.109405351s delta_ub 18446744073.698237236s) v5 -- 0x558e83e69c00 con 0x558e817fe400 2019-12-07T19:34:08.654+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.109405351s delta_ub 0.019443140s) v5 -- 0x558e83e69e00 con 0x558e817fe800 2019-12-07T19:34:08.654+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.109405351s delta_ub 0.019443140s) v5 -- 0x558e845c2000 con 0x558e817fec00 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6816/37076 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.112384769s delta_ub 0.002979418s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84053000 con 0x558e817ff800 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 25 osd.0 31 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:07.554661+0000 -> 2019-12-07T19:34:08.654942+0000 last_rx_front 2019-12-07T19:34:07.554715+0000 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 10 osd.0 31 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 29 -> 31 (as per caller) 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002751085s,0.002979418s]) 2019-12-07T19:34:08.654+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6814/37076 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.112420700s delta_ub 0.003015349s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8400 con 0x558e817ffc00 2019-12-07T19:34:08.654+0000 7fe2e753b700 25 osd.0 31 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.654942+0000 last_rx_front 2019-12-07T19:34:07.554715+0000 -> 2019-12-07T19:34:08.654980+0000 2019-12-07T19:34:08.654+0000 7fe2e753b700 25 osd.0 31 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:08.654713+0000) and older pending ping(s) 2019-12-07T19:34:08.654+0000 7fe2e753b700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002748883s,0.003015349s]) 2019-12-07T19:34:08.654+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6806/37077 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.126649602s delta_ub 0.017244251s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e817ff400 2019-12-07T19:34:08.654+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6808/37077 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.126600962s delta_ub 0.017195611s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8400 con 0x558e817ff000 2019-12-07T19:34:08.654+0000 7fe2e6d3a700 25 osd.0 31 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:07.554614+0000 last_rx_front 2019-12-07T19:34:07.554698+0000 -> 2019-12-07T19:34:08.655013+0000 2019-12-07T19:34:08.654+0000 7fe2e6d3a700 10 osd.0 31 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 29 -> 31 (as per caller) 2019-12-07T19:34:08.654+0000 7fe2e6d3a700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016944325s,0.017244251s]) 2019-12-07T19:34:08.654+0000 7fe2e753b700 25 osd.0 31 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:07.554614+0000 -> 2019-12-07T19:34:08.655029+0000 last_rx_front 2019-12-07T19:34:08.655013+0000 2019-12-07T19:34:08.654+0000 7fe2e753b700 25 osd.0 31 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:08.654713+0000) and older pending ping(s) 2019-12-07T19:34:08.654+0000 7fe2e753b700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016879463s,0.017195611s]) 2019-12-07T19:34:08.654+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6827/37078 5 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.111891475s delta_ub 0.002486124s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e818d7400 2019-12-07T19:34:08.654+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6831/37078 5 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.111843688s delta_ub 0.002438337s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8400 con 0x558e818d7800 2019-12-07T19:34:08.654+0000 7fe2e6d3a700 25 osd.0 31 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:07.554672+0000 last_rx_front 2019-12-07T19:34:07.554641+0000 -> 2019-12-07T19:34:08.655052+0000 2019-12-07T19:34:08.654+0000 7fe2e6d3a700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002147374s,0.002486124s]) 2019-12-07T19:34:08.654+0000 7fe2e753b700 25 osd.0 31 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:07.554672+0000 -> 2019-12-07T19:34:08.655063+0000 last_rx_front 2019-12-07T19:34:08.655052+0000 2019-12-07T19:34:08.654+0000 7fe2e753b700 25 osd.0 31 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:08.654713+0000) and older pending ping(s) 2019-12-07T19:34:08.654+0000 7fe2e753b700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002088162s,0.002438337s]) 2019-12-07T19:34:08.654+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6830/36939 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.090374596s delta_ub 18446744073.690520861s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e817fe800 2019-12-07T19:34:08.654+0000 7fe2e6d3a700 25 osd.0 31 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:07.554811+0000 -> 2019-12-07T19:34:08.655172+0000 last_rx_front 2019-12-07T19:34:07.554891+0000 2019-12-07T19:34:08.654+0000 7fe2e6d3a700 10 osd.0 31 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 29 -> 31 (as per caller) 2019-12-07T19:34:08.654+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6804/36936 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.121301881s delta_ub 0.011896530s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8400 con 0x558e817fe400 2019-12-07T19:34:08.654+0000 7fe2e6d3a700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690062216s,18446744073.690520861s]) 2019-12-07T19:34:08.654+0000 7fe2e753b700 25 osd.0 31 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:07.554864+0000 last_rx_front 2019-12-07T19:34:07.554854+0000 -> 2019-12-07T19:34:08.655192+0000 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6828/36939 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.090437426s delta_ub 18446744073.690583691s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84053000 con 0x558e817fec00 2019-12-07T19:34:08.654+0000 7fe2e753b700 10 osd.0 31 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 30 -> 31 (as per caller) 2019-12-07T19:34:08.654+0000 7fe2e753b700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011417827s,0.011896530s]) 2019-12-07T19:34:08.654+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6814/36938 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.104138294s delta_ub 18446744073.704284559s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e8181a000 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 25 osd.0 31 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.655172+0000 last_rx_front 2019-12-07T19:34:07.554891+0000 -> 2019-12-07T19:34:08.655204+0000 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 25 osd.0 31 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:34:08.654713+0000) and older pending ping(s) 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690092810s,18446744073.690583691s]) 2019-12-07T19:34:08.654+0000 7fe2e6d3a700 25 osd.0 31 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:07.554842+0000 -> 2019-12-07T19:34:08.655222+0000 last_rx_front 2019-12-07T19:34:07.554911+0000 2019-12-07T19:34:08.654+0000 7fe2e6d3a700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703776020s,18446744073.704284559s]) 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6812/36938 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.104213830s delta_ub 18446744073.704360095s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84053000 con 0x558e8181a400 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 25 osd.0 31 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.655222+0000 last_rx_front 2019-12-07T19:34:07.554911+0000 -> 2019-12-07T19:34:08.655237+0000 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 25 osd.0 31 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:08.654713+0000) and older pending ping(s) 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703836295s,18446744073.704360095s]) 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6806/36936 9 ==== osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:08.654713+0000/32.109405351s send_stamp 32.121396279s delta_ub 0.011990928s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84053000 con 0x558e817fe000 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 25 osd.0 31 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:07.554864+0000 -> 2019-12-07T19:34:08.655297+0000 last_rx_front 2019-12-07T19:34:08.655192+0000 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 25 osd.0 31 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:08.654713+0000) and older pending ping(s) 2019-12-07T19:34:08.654+0000 7fe2e7d3c700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011407554s,0.011990928s]) 2019-12-07T19:34:08.673+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.723+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.773+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.818+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:08.823+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.874+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.924+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:08.974+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.013+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:09.024+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.074+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.124+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.174+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.224+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.273+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.7 172.21.15.109:0/36939 9 ==== osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.709089891s delta_ub 18446744073.690583691s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e81749000 2019-12-07T19:34:09.273+0000 7fe2e6d3a700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.689903678s,18446744073.690583691s]) 2019-12-07T19:34:09.273+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.728737829s delta_ub 0.019647938s) v5 -- 0x558e83e69e00 con 0x558e81749000 2019-12-07T19:34:09.273+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.7 172.21.15.109:0/36939 9 ==== osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.709089891s delta_ub 18446744073.690583691s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e81748c00 2019-12-07T19:34:09.273+0000 7fe2e6d3a700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.689855822s,18446744073.690583691s]) 2019-12-07T19:34:09.273+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.273650+0000/32.709089891s send_stamp 32.728785685s delta_ub 0.019695794s) v5 -- 0x558e83e69600 con 0x558e81748c00 2019-12-07T19:34:09.274+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.324+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.353+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.1 172.21.15.131:0/37078 4 ==== osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.354397+0000/32.811320781s send_stamp 32.811320781s delta_ub 0.002486124s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84053000 con 0x558e81985000 2019-12-07T19:34:09.353+0000 7fe2e7d3c700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002071954s,0.002486124s]) 2019-12-07T19:34:09.353+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.354397+0000/32.811320781s send_stamp 32.809248827s delta_ub 18446744073.707479662s) v5 -- 0x558e845c2000 con 0x558e81985000 2019-12-07T19:34:09.353+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.1 172.21.15.131:0/37078 4 ==== osd_ping(ping e31 up_from 13 ping_stamp 2019-12-07T19:34:09.354397+0000/32.811320781s send_stamp 32.811320781s delta_ub 0.002486124s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee800 con 0x558e81984c00 2019-12-07T19:34:09.353+0000 7fe2e6d3a700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002012374s,0.002486124s]) 2019-12-07T19:34:09.353+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.354397+0000/32.811320781s send_stamp 32.809308407s delta_ub 18446744073.707539242s) v5 -- 0x558e83e69600 con 0x558e81984c00 2019-12-07T19:34:09.375+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.425+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.475+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.484+0000 7fe2e2ca2700 10 osd.0 31 tick_without_osd_lock 2019-12-07T19:34:09.484+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fad3000/0x40000000/0x1680000000, data 0x4f857c/0x52b000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:09.484+0000 7fe2e2ca2700 25 osd.0 31 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.655063+0000 last_rx_front 2019-12-07T19:34:08.655052+0000 2019-12-07T19:34:09.484+0000 7fe2e2ca2700 25 osd.0 31 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.655029+0000 last_rx_front 2019-12-07T19:34:08.655013+0000 2019-12-07T19:34:09.484+0000 7fe2e2ca2700 25 osd.0 31 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.654942+0000 last_rx_front 2019-12-07T19:34:08.654980+0000 2019-12-07T19:34:09.484+0000 7fe2e2ca2700 25 osd.0 31 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.655222+0000 last_rx_front 2019-12-07T19:34:08.655237+0000 2019-12-07T19:34:09.484+0000 7fe2e2ca2700 25 osd.0 31 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.655297+0000 last_rx_front 2019-12-07T19:34:08.655192+0000 2019-12-07T19:34:09.484+0000 7fe2e2ca2700 25 osd.0 31 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.655172+0000 last_rx_front 2019-12-07T19:34:08.655204+0000 2019-12-07T19:34:09.484+0000 7fe2e2ca2700 20 osd.0 31 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:09.484+0000 7fe2e2ca2700 10 osd.0 31 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:09.484+0000 7fe2e2ca2700 20 osd.0 31 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:09.484+0000 7fe2e2ca2700 10 osd.0 31 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:09.525+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.575+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.594+0000 7fe2e4535700 10 osd.0 31 tick 2019-12-07T19:34:09.594+0000 7fe2e4535700 10 osd.0 31 do_waiters -- start 2019-12-07T19:34:09.594+0000 7fe2e4535700 10 osd.0 31 do_waiters -- finish 2019-12-07T19:34:09.594+0000 7fe2e4535700 20 osd.0 31 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:09.606+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 128 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7000 2019-12-07T19:34:09.607+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.1s1 epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:09.607+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.607+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.607+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.607+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.607+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e32) pg 0x558e819e4000 2019-12-07T19:34:09.607+0000 7fe2cb26d700 20 osd.0 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 32, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.607+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:09.607+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 47 ==== pg_info2(1.ds5 1.ds5 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e84704000 con 0x558e818d6000 2019-12-07T19:34:09.607+0000 7fe2e753b700 15 osd.0 31 enqueue_peering_evt 1.ds5 epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:09.607+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.607+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 129 ==== pg_info2(1.fs1 1.fs1 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7000 2019-12-07T19:34:09.607+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.fs1 epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:09.607+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.607+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.607+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.607+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.607+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e32) pg 0x558e81a9b000 2019-12-07T19:34:09.607+0000 7fe2c8267700 20 osd.0 op_wq _add_slot_waiter 1.fs1 peering, item epoch is 32, will wait on OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.607+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:09.607+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.607+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.607+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.607+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e32) pg 0x558e819e7000 2019-12-07T19:34:09.607+0000 7fe2c9269700 20 osd.0 op_wq _add_slot_waiter 1.ds5 peering, item epoch is 32, will wait on OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.607+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:09.607+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 48 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e84704000 con 0x558e818d6000 2019-12-07T19:34:09.607+0000 7fe2e753b700 15 osd.0 31 enqueue_peering_evt 1.5s4 epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:09.607+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.607+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.607+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.607+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.607+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e32) pg 0x558e83d07000 2019-12-07T19:34:09.607+0000 7fe2c5261700 20 osd.0 op_wq _add_slot_waiter 1.5s4 peering, item epoch is 32, will wait on OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.607+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:09.609+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 129 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c46e00 con 0x558e818fe400 2019-12-07T19:34:09.609+0000 7fe2e6d3a700 15 osd.0 31 enqueue_peering_evt 1.bs1 epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:09.609+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.609+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e32) pg 0x558e83c78000 2019-12-07T19:34:09.609+0000 7fe2ca26b700 20 osd.0 op_wq _add_slot_waiter 1.bs1 peering, item epoch is 32, will wait on OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:09.609+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 130 ==== pg_info2(1.3s5 1.3s5 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c46e00 con 0x558e818fe400 2019-12-07T19:34:09.609+0000 7fe2e6d3a700 15 osd.0 31 enqueue_peering_evt 1.3s5 epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:09.609+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.609+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.609+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e32) pg 0x558e81231000 2019-12-07T19:34:09.609+0000 7fe2c6263700 20 osd.0 op_wq _add_slot_waiter 1.3s5 peering, item epoch is 32, will wait on OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.609+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:09.612+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 40 ==== pg_info2(1.4s1 1.4s1 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e84704000 con 0x558e818fe000 2019-12-07T19:34:09.612+0000 7fe2e753b700 15 osd.0 31 enqueue_peering_evt 1.4s1 epoch_sent: 32 epoch_requested: 32 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s) 2019-12-07T19:34:09.612+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.612+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.612+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.612+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.612+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e32) pg 0x558e81938000 2019-12-07T19:34:09.612+0000 7fe2c5a62700 20 osd.0 op_wq _add_slot_waiter 1.4s1 peering, item epoch is 32, will wait on OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.612+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:09.614+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 241 ==== pg_info2(1.9s3 1.9s3 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:09.614+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.9s3 epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:09.614+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.614+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.614+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e32) pg 0x558e8192b000 2019-12-07T19:34:09.614+0000 7fe2c7265700 20 osd.0 op_wq _add_slot_waiter 1.9s3 peering, item epoch is 32, will wait on OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.614+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:09.614+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 242 ==== pg_info2(1.as3 1.as3 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:09.614+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.as3 epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:09.614+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e32) pg 0x558e8193b000 2019-12-07T19:34:09.614+0000 7fe2caa6c700 20 osd.0 op_wq _add_slot_waiter 1.as3 peering, item epoch is 32, will wait on OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.614+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 243 ==== pg_info2(1.7s3 1.7s3 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:09.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:09.614+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.7s3 epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:09.614+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.614+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.614+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.614+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e32) pg 0x558e8122e000 2019-12-07T19:34:09.614+0000 7fe2c425f700 20 osd.0 op_wq _add_slot_waiter 1.7s3 peering, item epoch is 32, will wait on OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.614+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:09.615+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 244 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:09.615+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.es4 epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:09.615+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.615+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.615+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.615+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.615+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e32) pg 0x558e81a98000 2019-12-07T19:34:09.615+0000 7fe2c4a60700 20 osd.0 op_wq _add_slot_waiter 1.es4 peering, item epoch is 32, will wait on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.615+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:09.617+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 245 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 ==== 1076+0+0 (unknown 0 0 0) 0x558e83c48300 con 0x558e818d7c00 2019-12-07T19:34:09.617+0000 7fe2e7d3c700 15 osd.0 31 enqueue_peering_evt 1.8s5 epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:09.617+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e32) pg 0x558e841a4000 2019-12-07T19:34:09.617+0000 7fe2c7a66700 20 osd.0 op_wq _add_slot_waiter 1.8s5 peering, item epoch is 32, will wait on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e32) 2019-12-07T19:34:09.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:09.625+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 1 osrs, 3 txcs 2019-12-07T19:34:09.625+0000 7fe2dc28f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock osr 0x558e8107a500 2 ios pending 2019-12-07T19:34:09.625+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 21 0x2000~1000 2019-12-07T19:34:09.625+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x2000~1000 crc 306374a5 2019-12-07T19:34:09.625+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 21 0x1b000~1000 2019-12-07T19:34:09.625+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x1b000~1000 crc e2b79a84 2019-12-07T19:34:09.625+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.625+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish osr 0x558e8107a500 2019-12-07T19:34:09.625+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish dequeueing 2019-12-07T19:34:09.675+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.725+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 531597 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.760+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.4 172.21.15.109:0/36938 4 ==== osd_ping(ping e32 up_from 13 ping_stamp 2019-12-07T19:34:09.761083+0000/33.210309283s send_stamp 33.210309283s delta_ub 18446744073.704360095s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83e68800 con 0x558e81985800 2019-12-07T19:34:09.760+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.4 172.21.15.109:0/36938 4 ==== osd_ping(ping e32 up_from 13 ping_stamp 2019-12-07T19:34:09.761083+0000/33.210309283s send_stamp 33.210309283s delta_ub 18446744073.704360095s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e81985400 2019-12-07T19:34:09.760+0000 7fe2e7d3c700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703761997s,18446744073.704360095s]) 2019-12-07T19:34:09.760+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.761083+0000/33.210309283s send_stamp 33.216098902s delta_ub 0.005789619s) v5 -- 0x558e845c2000 con 0x558e81985800 2019-12-07T19:34:09.760+0000 7fe2e7d3c700 10 osd.0 31 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 31 -> 32 (as per caller) 2019-12-07T19:34:09.760+0000 7fe2e6d3a700 20 osd.0 31 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703719987s,18446744073.704360095s]) 2019-12-07T19:34:09.760+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e31 up_from 13 ping_stamp 2019-12-07T19:34:09.761083+0000/33.210309283s send_stamp 33.216140912s delta_ub 0.005831629s) v5 -- 0x558e83e69600 con 0x558e81985400 2019-12-07T19:34:09.760+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 131 ==== osd_map(32..32 src has 1..32) v4 ==== 661+0+0 (unknown 0 0 0) 0x558e846e26c0 con 0x558e818fe400 2019-12-07T19:34:09.760+0000 7fe2d4a80700 20 osd.0 31 OSD::ms_dispatch: osd_map(32..32 src has 1..32) v4 2019-12-07T19:34:09.760+0000 7fe2d4a80700 10 osd.0 31 do_waiters -- start 2019-12-07T19:34:09.760+0000 7fe2d4a80700 10 osd.0 31 do_waiters -- finish 2019-12-07T19:34:09.760+0000 7fe2d4a80700 20 osd.0 31 _dispatch 0x558e846e26c0 osd_map(32..32 src has 1..32) v4 2019-12-07T19:34:09.761+0000 7fe2d4a80700 3 osd.0 31 handle_osd_map epochs [32,32], i have 31, src has [1,32] 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 osd.0 31 handle_osd_map got inc map for epoch 32 2019-12-07T19:34:09.761+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3398e826:::osdmap.31:0# 0x0~0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1470 size 0x1470 (5232) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841903f0 blob([0x7a000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f95ab0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95b20 blob([!~1000,0x7b000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,470]) SharedBlob(0x558e84549ab0 sbid 0x0)) need 0x1000~470 cache has 0x[1000~470] 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3398e826:::osdmap.31:0# 0x0~1470 = 5232 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 osd.0 31 add_map_bl 31 5232 bytes 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 osd.0 31 got_full_map 32, nothing requested 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 osd.0 31 handle_osd_map pg_num_history pg_num_history(e32 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps purged_snaps {32={}} 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 osd.0 31 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e32 [1,32] lci=[13,32]) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e83e72600 seq 24 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c18b972f:::inc_osdmap.32:0# key 0x7f7fffffffffffffffc18b97'/!inc_osdmap.32!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c18b972f:::inc_osdmap.32:0# 0x0~269 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1154 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c18b972f:::inc_osdmap.32:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~269 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72600 1 blobs 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x2b5000~1000/1000,1000,0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x2b5000~1000] 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84549ce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c3f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84549ce0 blob([0x2b5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c3f0 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.blob(0x558e84549ce0) get_ref 0x0~269 Blob(0x558e84549ce0 blob([0x2b5000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c3f0 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.blob(0x558e84549ce0) get_ref init 0x1000, 1000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558e84549ce0 blob([0x2b5000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558e83e6c3f0 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e83e6c408 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x269 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8417c608) dirty_range mark inline shard dirty 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c18b972f:::inc_osdmap.32:0# 0x0~269 = 0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3978e826:::osdmap.32:0# key 0x7f7fffffffffffffff3978e8'&!osdmap.32!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3978e826:::osdmap.32:0# 0x0~1454 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1155 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3978e826:::osdmap.32:0# 0x0~1454 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~454 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbac on front/back, now 0x0~1000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72600 2 blobs 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x2b6000~2000/1000,2000,0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x2b6000~2000] 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f61110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e841900e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f61110 blob([0x2b6000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841900e0 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.blob(0x558e83f61110) get_ref 0x0~1000 Blob(0x558e83f61110 blob([0x2b6000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841900e0 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.blob(0x558e83f61110) get_ref init 0x1000, 1000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e83f61110 blob([0x2b6000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e841900e0 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e841900f8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e847556c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84755650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e847556c0 blob([!~1000,0x2b7000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755650 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.blob(0x558e847556c0) get_ref 0x1000~454 Blob(0x558e847556c0 blob([!~1000,0x2b7000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e84755650 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.blob(0x558e847556c0) get_ref init 0x2000, 1000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~454: 0x1000~454 Blob(0x558e847556c0 blob([!~1000,0x2b7000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558e84755650 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e84755668 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1454 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8417da08) dirty_range mark inline shard dirty 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3978e826:::osdmap.32:0# 0x0~1454 = 0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72600 0 blobs 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e72600 0 blobs 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72600 cost 23302 (4 ios * 4000 + 7302 bytes) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72600 onodes 0x558e80251b80,0x558e817b9180,0x558e8417c500,0x558e8417d900 shared_blobs 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8417c608) update #-1:c18b972f:::inc_osdmap.32:0# 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8417c608) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c18b972f:::inc_osdmap.32:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8417da08) update #-1:3978e826:::osdmap.32:0# 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8417da08) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3978e826:::osdmap.32:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72600 allocated 0x[2b5000~3000] released 0x[] 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 freelist allocate 0x2b5000~3000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72600 prepare 2019-12-07T19:34:09.761+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83e72600 2019-12-07T19:34:09.761+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72600 aio_wait 2019-12-07T19:34:09.761+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72600 2019-12-07T19:34:09.761+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72600 io_done 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3078 < min_flush_size 524288 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8800 < min_flush_size 524288 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c500 had 1 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417d900 had 1 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3859677184 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998491) bluestore 86 GiB free (0.959943), bluefs_ratio 0.0399447 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8859 < min_flush_size 524288 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8859 < min_flush_size 524288 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x11f3fa~229b to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x11f3fa 0x11f3fa~229b to file(ino 16 size 0x120000 mtime 2019-12-07T19:34:06.699076+0000 allocated 120000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~f0000]) 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs _allocate len 0x1695 from 0 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs _allocate len 0x1695 from 1 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd250000 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd250000~10000/10000,10000,acd250000 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x130000 to include allocated 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 37 (was clean) 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xef3fa 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x3fa 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x695 and padding block with 0x96b 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x130000 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x121695 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (37) on file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]), flushing log 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 37 len 0x49 crc 0x2a2fb65c) 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x24000~1000 to file(ino 1 size 0x24000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:09.761+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x24000 0x24000~1000 to file(ino 1 size 0x24000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x25000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:09.761+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x24000 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x25000 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000017 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 37 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 1 in 0.000452783s (0.000005691s flush + 0.000447092s kv commit) 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e72600> 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <0x558e846eb040> 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72600 kv_submitted 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72600 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e83e72600 osr 0x558e8107a500 2019-12-07T19:34:09.762+0000 7fe2d7285700 10 osd.0 31 _committed_osd_maps 32..32 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e845eedc0) prepare_write seq 22 0x1b000~1000 crc 8d83060d 2019-12-07T19:34:09.762+0000 7fe2d7285700 10 osd.0 31 advance to epoch 32 (<= last 32 <= newest_map 32) 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e845eedc0) prepare_write seq 22 0x2000~1000 crc 66b548f4 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 deferred_cleanup 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72000 onodes 0x558e80251b80,0x558e817b9180,0x558e8191bb80,0x558e83c73680 2019-12-07T19:34:09.762+0000 7fe2d7285700 7 osd.0 32 consume_map version 32 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7b9e8 in 0x558e80219340) _finish_write discard buffer(0x558e846daa20 space 0x558e81a7b9e8 0x0~1000 writing nocache) 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 32 identify_splits_and_merges 1.8s5 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8423c7f8 in 0x558e80219340) _finish_write discard buffer(0x558e846daab0 space 0x558e8423c7f8 0x1000~1000 writing nocache) 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 32 identify_splits_and_merges 1.9s3 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 32 identify_splits_and_merges 1.1s1 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84549c18 in 0x558e80219340) _finish_write discard buffer(0x558e80229dd0 space 0x558e84549c18 0x0~1000 writing nocache) 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 32 identify_splits_and_merges 1.as3 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 32 identify_splits_and_merges 1.2s0 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72000 done 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 32 identify_splits_and_merges 1.3s5 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72300 deferred_cleanup 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 32 identify_splits_and_merges 1.bs1 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72000 [] 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 32 identify_splits_and_merges 1.4s1 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 32 identify_splits_and_merges 1.5s4 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 32 identify_splits_and_merges 1.ds5 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 32 identify_splits_and_merges 1.es4 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 32 identify_splits_and_merges 1.7s3 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 32 identify_splits_and_merges 1.fs1 e31 to e32 pg_nums {18=16} 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 32 prune_pg_created 2019-12-07T19:34:09.762+0000 7fe2d7285700 10 osd.0:0.consume_map 32 (was 31) 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72300 deferred_cleanup 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72300 onodes 0x558e80251b80,0x558e811dcc80,0x558e817b9180,0x558e819b6280 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6d4a8 in 0x558e80219340) _finish_write discard buffer(0x558e846dad80 space 0x558e83e6d4a8 0x1000~1000 writing nocache) 2019-12-07T19:34:09.762+0000 7fe2d7285700 10 osd.0:1.consume_map 32 (was 31) 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.9s3 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e845499e8 in 0x558e80219340) _finish_write discard buffer(0x558e846dacf0 space 0x558e845499e8 0x0~1000 writing nocache) 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2d7285700 10 osd.0:2.consume_map 32 (was 31) 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846c4398 in 0x558e80219340) _finish_write discard buffer(0x558e846dac60 space 0x558e846c4398 0x0~1000 writing nocache) 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.as3 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72300 done 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e32) pg 0x558e841a4000 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa900 deferred_cleanup 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72300 [] 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.762+0000 7fe2d7285700 10 osd.0:3.consume_map 32 (was 31) 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fa900 deferred_cleanup 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.3s5 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fa900 onodes 0x558e80251b80,0x558e817b9180,0x558e83f29180,0x558e83f29400 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f95ac8 in 0x558e80219340) _finish_write discard buffer(0x558e84032750 space 0x558e83f95ac8 0x0~1000 writing nocache) 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:3._wake_pg_slot _wake_pg_slot 1.bs1 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e32) pg 0x558e819e4000 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84190788 in 0x558e80219340) _finish_write discard buffer(0x558e840326c0 space 0x558e84190788 0x0~1000 writing nocache) 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84549ac8 in 0x558e80219340) _finish_write discard buffer(0x558e840327e0 space 0x558e84549ac8 0x1000~1000 writing nocache) 2019-12-07T19:34:09.762+0000 7fe2d7285700 10 osd.0:4.consume_map 32 (was 31) 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fa900 done 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72600 deferred_queued 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fa900 [] 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e32) pg 0x558e8193b000 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:4._wake_pg_slot _wake_pg_slot 1.4s1 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2d7285700 10 osd.0:5.consume_map 32 (was 31) 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e32) pg 0x558e83c78000 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:5._wake_pg_slot _wake_pg_slot 1.5s4 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0:0.update_pg_epoch 1.8s5 31 -> 32 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.1s1 31 -> 32 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:5._wake_pg_slot _wake_pg_slot 1.ds5 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 osd.0 pg_epoch: 31 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] handle_advance_map [1,4,5,2,3,0]/[1,4,5,2,3,0] -- 1/1 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2d7285700 10 osd.0:6.consume_map 32 (was 31) 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,7~1]] handle_advance_map [3,0,2,4,5,1]/[3,0,2,4,5,1] -- 3/3 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.as3 31 -> 32 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e32) pg 0x558e81938000 2019-12-07T19:34:09.762+0000 7fe2d7285700 10 osd.0:7.consume_map 32 (was 31) 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 osd.0 pg_epoch: 32 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs1 31 -> 32 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.7s3 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 31 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] handle_advance_map [1,4,2,0,5,3]/[1,4,2,0,5,3] -- 1/1 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 pending_peering first epoch 32 <= 32, requeueing 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 osd.0 pg_epoch: 32 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.fs1 to_process <> waiting <> waiting_peering {32=} 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:09.762+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.8s5_head options 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.762+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:34:09.762+0000 7fe2d7285700 10 osd.0 32 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 osd.0 pg_epoch: 32 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.762+0000 7fe2d7285700 15 osd.0 32 enqueue_peering_evt 1.8s5 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.762+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 osd.0:4.update_pg_epoch 1.4s1 31 -> 32 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0 pg_epoch: 32 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.762+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs1_head options 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0 pg_epoch: 32 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.762+0000 7fe2d7285700 15 osd.0 32 enqueue_peering_evt 1.9s3 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 31 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.762+0000 7fe2d7285700 15 osd.0 32 enqueue_peering_evt 1.1s1 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 pg_epoch: 32 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 osd.0 pg_epoch: 32 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.762+0000 7fe2d7285700 15 osd.0 32 enqueue_peering_evt 1.as3 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 32 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 pg_epoch: 32 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 osd.0 pg_epoch: 32 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 32 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.762+0000 7fe2d7285700 15 osd.0 32 enqueue_peering_evt 1.2s0 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0 pg_epoch: 32 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2d7285700 15 osd.0 32 enqueue_peering_evt 1.3s5 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 32 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.762+0000 7fe2d7285700 15 osd.0 32 enqueue_peering_evt 1.bs1 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 osd.0 pg_epoch: 32 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.762+0000 7fe2d7285700 15 osd.0 32 enqueue_peering_evt 1.4s1 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 32 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 pg_epoch: 32 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 osd.0 pg_epoch: 32 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2d7285700 15 osd.0 32 enqueue_peering_evt 1.5s4 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 32 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 osd.0 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.762+0000 7fe2d7285700 15 osd.0 32 enqueue_peering_evt 1.ds5 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.762+0000 7fe2d7285700 15 osd.0 32 enqueue_peering_evt 1.es4 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 32 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.762+0000 7fe2d7285700 15 osd.0 32 enqueue_peering_evt 1.7s3 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 32 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 32 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e83c57800 seq 39 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e8195ef00 seq 31 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e8417ac00 seq 23 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 32 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s) 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e819a1b00 seq 16 2019-12-07T19:34:09.762+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e32) pg 0x558e81a9b000 2019-12-07T19:34:09.762+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._biginfo' <- _biginfo 2019-12-07T19:34:09.762+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 32 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._epoch' <- _epoch 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._biginfo' <- _biginfo 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._epoch' <- _epoch 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83c57800 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83c57800 onodes shared_blobs 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195ef00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1b00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195ef00 onodes shared_blobs 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1b00 onodes shared_blobs 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1b00 allocated 0x[] released 0x[] 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417ac00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.fs1 31 -> 32 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195ef00 allocated 0x[] released 0x[] 2019-12-07T19:34:09.762+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417ac00 onodes shared_blobs 2019-12-07T19:34:09.762+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417ac00 allocated 0x[] released 0x[] 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2d7285700 15 osd.0 32 enqueue_peering_evt 1.fs1 epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83c57800 allocated 0x[] released 0x[] 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 31 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,7~1]] handle_advance_map [3,0,5,4,1,2]/[3,0,5,4,1,2] -- 3/3 2019-12-07T19:34:09.762+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e32) pg 0x558e819e7000 2019-12-07T19:34:09.762+0000 7fe2d7285700 7 osd.0 32 activate_map version 32 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 prepare 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 prepare 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 prepare 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83c57800 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1b00 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 io_done 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417ac00 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 io_done 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 io_done 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 prepare 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195ef00 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 io_done 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.ds5 31 -> 32 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558e841a4000 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558e8193b000 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 31 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e32) pg 0x558e8192b000 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 pg_epoch: 32 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 osd.0 pg_epoch: 32 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 pg_epoch: 32 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.762+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558e81075000 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.762+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2cc26f700 20 osd.0 32 agent_entry empty queue 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x121695~8f3 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.9s3 31 -> 32 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 pg_epoch: 32 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 pg_epoch: 32 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x121695 0x121695~8f3 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 pg_epoch: 32 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf1695 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 31 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x695 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.2s0 31 -> 32 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xf88 and padding block with 0x78 2019-12-07T19:34:09.762+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 31 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,7~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.762+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,7~1]] state: Active advmap 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 pg_epoch: 32 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e81a0bb00 seq 37 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] on_active_advmap new removed_snaps [6~1], snap_trimq now [6~1] 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 pg_epoch: 32 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.762+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 pg_epoch: 32 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x121f88 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e32) pg 0x558e81231000 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:09.762+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0bb00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0bb00 onodes shared_blobs 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e845d4900 seq 20 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000105382s (0.000000070s flush + 0.000105312s kv commit) 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 pg_epoch: 32 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.762+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 prepare 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0bb00 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 io_done 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.9s0( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.3s5 31 -> 32 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845d4900 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e32) pg 0x558e8122e000 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e32) pg 0x558e81a98000 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 31 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e8195e300 seq 13 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] state: Active: handling ActMap 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2291 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.7s3 31 -> 32 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] Active: kicking snap trim 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e84558f00 seq 34 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 osd.0:6.update_pg_epoch 1.es4 31 -> 32 2019-12-07T19:34:09.762+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.3s5_head options 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e300 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x121f88~8f3 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 31 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e300 onodes shared_blobs 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e300 allocated 0x[] released 0x[] 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x121f88 0x121f88~8f3 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.762+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 osd.0 pg_epoch: 31 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,7~1]] handle_advance_map [1,3,4,5,0,2]/[1,3,4,5,0,2] -- 1/1 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf1f88 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xf88 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 prepare 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 pg_epoch: 32 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 osd.0 pg_epoch: 32 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e300 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] exit NotTrimming 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83c57800,0x558e8417ac00> 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 io_done 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 pg_epoch: 32 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 kv_submitted 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84558f00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83c57800 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84558f00 onodes shared_blobs 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 osd.0 pg_epoch: 32 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] enter Trimming 2019-12-07T19:34:09.762+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84558f00 allocated 0x[] released 0x[] 2019-12-07T19:34:09.762+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:34:09.762+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83c57800 finishing 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83c57800 onodes 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845d4900 onodes shared_blobs 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83c57800 done 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845d4900 allocated 0x[] released 0x[] 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558f00 prepare 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 osd.0 pg_epoch: 32 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] enter Trimming/WaitReservation 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.762+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4900 prepare 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83c57800 [] 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 osd.0 pg_epoch: 32 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845d4900 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 pg_epoch: 32 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4900 io_done 2019-12-07T19:34:09.762+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558e81938000 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 osd.0 pg_epoch: 32 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 pg_epoch: 32 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 pg_epoch: 32 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 kv_submitted 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 osd.0 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417ac00 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417ac00 finishing 2019-12-07T19:34:09.762+0000 7fe2c5a62700 10 osd.0 pg_epoch: 32 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417ac00 onodes 2019-12-07T19:34:09.762+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417ac00 done 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417ac00 [] 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.762+0000 7fe2c6a64700 15 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] publish_stats_to_osd 31:95 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 osd.0 pg_epoch: 32 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e32) pg 0x558e83d07000 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84558f00 2019-12-07T19:34:09.762+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558f00 io_done 2019-12-07T19:34:09.762+0000 7fe2c6a64700 7 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] state: handle ActMap primary 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 osd.0 pg_epoch: 32 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.762+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 osd.0 pg_epoch: 32 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x87b and padding block with 0x785 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 pg_epoch: 32 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558e8192b000 2019-12-07T19:34:09.762+0000 7fe2c6a64700 15 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] publish_stats_to_osd 31:96 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 osd.0 pg_epoch: 32 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e819a1200 seq 16 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,7~1]] handle_advance_map: 32 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.5s4 31 -> 32 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 osd.0 pg_epoch: 32 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._biginfo' <- _biginfo 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 31 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,7~1]] handle_advance_map [2,1,3,4,0,5]/[2,1,3,4,0,5] -- 2/2 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558e819e4000 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._epoch' <- _epoch 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,7~1]] state: Started advmap 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1200 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.762+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1200 onodes shared_blobs 2019-12-07T19:34:09.762+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e81a0b500 seq 37 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e811fbb00 seq 27 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,7~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1200 allocated 0x[] released 0x[] 2019-12-07T19:34:09.762+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.5s4_head options 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:09.762+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 prepare 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,7~1]] plpg_on_pool_change 2019-12-07T19:34:09.762+0000 7fe2cea74700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1200 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 io_done 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 pg_epoch: 32 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,7~1]] hit_set_clear 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fbb00 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.762+0000 7fe2cea74700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: NotTrimming: trimming 6 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fbb00 onodes shared_blobs 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fbb00 allocated 0x[] released 0x[] 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e81a0b500 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 pg_epoch: 32 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,7~1]] agent_stop 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e81a0b500 onodes shared_blobs 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 prepare 2019-12-07T19:34:09.762+0000 7fe2cea74700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[6~1] ps=[4~1,7~1]] exit Trimming/WaitReservation 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 32 advance_pg new pool opts old pool opts 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e81a0b500 allocated 0x[] released 0x[] 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fbb00 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558e81231000 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 io_done 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,7~1]] handle_activate_map: 32 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 prepare 2019-12-07T19:34:09.762+0000 7fe2cea74700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e81a0b500 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558e81a98000 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 io_done 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,7~1]] activate_map 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2cea74700 15 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] publish_stats_to_osd 32:97 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 osd.0 pg_epoch: 32 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 pg_epoch: 32 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,7~1]] activate_map: Not dirtying info: last_persisted is 31 while current is 32 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2cea74700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558e8122e000 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558e83c78000 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,7~1]] do_peering_event: epoch_sent: 32 epoch_requested: 32 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:09.762+0000 7fe2cea74700 10 osd.0 32 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[6~1] ps=[4~1,7~1]] for snaptrim 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 osd.0 pg_epoch: 32 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2cea74700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=32) prio 5 cost 1048576 e32) 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,7~1]] proc_primary_info updating purged_snaps to [4~1,6~2] 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558e81a9b000 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=32) prio 5 cost 1048576 e32) queued 2019-12-07T19:34:09.762+0000 7fe2cea74700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] prepare_stats_for_publish reporting purged_snaps [4~1,7~1] 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 osd.0 pg_epoch: 32 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2cea74700 15 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] publish_stats_to_osd 32:98 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e845d4c00 seq 18 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=32) prio 5 cost 1048576 e32) pg 0x558e81075000 2019-12-07T19:34:09.762+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] snap_trimmer posting 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._biginfo' <- _biginfo 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._epoch' <- _epoch 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: AwaitAsyncWork: trimming snap 6 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845d4c00 cost 5461 (1 ios * 4000 + 1461 bytes) 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 1 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845d4c00 onodes shared_blobs 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845d4c00 allocated 0x[] released 0x[] 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4c00 prepare 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845d4c00 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4c00 io_done 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558e83d07000 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x12287b 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore.OmapIteratorImpl(0x558e819bd080) upper_bound after SNA_1_0000000000000006_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_0000000000000006_.0_0000000000000001.2' 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore.OmapIteratorImpl(0x558e819bd080) valid is at 0x00000000000000000000000000000402'~' 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) queued 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000283163s (0.000000076s flush + 0.000283087s kv commit) 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_0000000000000006_.0_0000000000000001.2) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 32 epoch_requested: 32 NullEvt) prio 255 cost 10 e32) pg 0x558e819e7000 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 8 submitting 8 deferred done 0 stable 0 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195ef00,0x558e819a1b00> 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 kv_submitted 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195ef00 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[6~1] ps=[4~1,7~1]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195ef00 finishing 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:09.762+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195ef00 onodes 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1,7~1]] SnapTrimmer state: adding snap 6 to purged_snaps 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195ef00 done 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2268 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 osd.0 pg_epoch: 32 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 32 epoch_requested: 32 NullEvt 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195ef00 [] 2019-12-07T19:34:09.762+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.762+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 kv_submitted 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1b00 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3402 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1b00 finishing 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1b00 onodes 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1b00 done 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1b00 [] 2019-12-07T19:34:09.762+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:34:09.762+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4536 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1,6~2]] SnapTrimmer state: purged_snaps now [4~1,6~2], snap_trimq now [] 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5670 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e845d4600 seq 40 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:09.762+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 6804 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 7938 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845d4600 cost 5291 (1 ios * 4000 + 1291 bytes) 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845d4600 onodes shared_blobs 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845d4600 allocated 0x[] released 0x[] 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 9242 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4600 prepare 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845d4600 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4600 io_done 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 9265 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 9265 < min_flush_size 524288 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1,6~2]] share_pg_info 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x12287b~2431 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.762+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x12287b 0x12287b~2431 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf287b 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x87b 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:09.762+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xcac and padding block with 0x354 2019-12-07T19:34:09.762+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 -- 0x558e83ebbc00 con 0x558e818d7c00 2019-12-07T19:34:09.762+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 -- 0x558e83ebd800 con 0x558e818d7000 2019-12-07T19:34:09.762+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 -- 0x558e83ebc300 con 0x558e818fe400 2019-12-07T19:34:09.762+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 -- 0x558e83eba000 con 0x558e818d6800 2019-12-07T19:34:09.762+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e32/32) v1 -- 0x558e83ebb500 con 0x558e818fe000 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1,6~2]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1,6~2]] exit Trimming 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:09.762+0000 7fe2c6a64700 15 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] publish_stats_to_osd 32:99 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] enter NotTrimming 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:09.762+0000 7fe2c6a64700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] snap_trimmer complete 2019-12-07T19:34:09.762+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:09.762+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x124cac 2019-12-07T19:34:09.763+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:09.763+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:09.763+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:09.763+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:09.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 8 cleaned 0 in 0.000376215s (0.000000063s flush + 0.000376152s kv commit) 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:09.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e81a0bb00,0x558e8195e300,0x558e845d4900,0x558e84558f00,0x558e819a1200,0x558e811fbb00,0x558e81a0b500,0x558e845d4c00> 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 kv_submitted 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0bb00 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0bb00 finishing 2019-12-07T19:34:09.763+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1134 < min_flush_size 524288 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0bb00 onodes 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0bb00 done 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0bb00 [] 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 kv_submitted 2019-12-07T19:34:09.763+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e300 2019-12-07T19:34:09.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e300 finishing 2019-12-07T19:34:09.763+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e300 onodes 2019-12-07T19:34:09.763+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1157 < min_flush_size 524288 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e300 done 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df900 q now empty 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e300 [] 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4900 kv_submitted 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845d4900 2019-12-07T19:34:09.763+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4900 finishing 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845d4900 onodes 2019-12-07T19:34:09.763+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x124cac~485 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845d4900 done 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845d4900 [] 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558f00 kv_submitted 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84558f00 2019-12-07T19:34:09.763+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x124cac 0x124cac~485 to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558f00 finishing 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84558f00 onodes 2019-12-07T19:34:09.763+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84558f00 done 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:09.763+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf4cac 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84558f00 [] 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.763+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xcac 2019-12-07T19:34:09.763+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 kv_submitted 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1200 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1200 finishing 2019-12-07T19:34:09.763+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1200 onodes 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1200 done 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:09.763+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1200 [] 2019-12-07T19:34:09.763+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.763+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x131 and padding block with 0xecf 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 kv_submitted 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fbb00 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fbb00 finishing 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fbb00 onodes 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fbb00 done 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fbb00 [] 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 kv_submitted 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e81a0b500 2019-12-07T19:34:09.763+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e81a0b500 finishing 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e81a0b500 onodes 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e81a0b500 done 2019-12-07T19:34:09.763+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e81a0b500 [] 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.763+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4c00 kv_submitted 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845d4c00 2019-12-07T19:34:09.763+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4c00 finishing 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845d4c00 onodes 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845d4c00 done 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845d4c00 [] 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:09.763+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x125131 2019-12-07T19:34:09.763+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:09.763+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:09.763+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:09.763+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:09.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000141406s (0.000000078s flush + 0.000141328s kv commit) 2019-12-07T19:34:09.763+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e845d4600> 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:09.763+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4600 kv_submitted 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845d4600 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4600 finishing 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845d4600 onodes 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845d4600 done 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845d4600 [] 2019-12-07T19:34:09.763+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:09.763+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:09.763+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:09.763+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:09.764+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:09.775+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.825+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.876+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.926+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:09.976+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.026+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.076+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.110+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83d41400 0x558e8405a680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:10.110+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83d41400 0x558e8405a680 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:10.110+0000 7fe2e753b700 10 osd.0 32 ms_handle_authentication new session 0x558e83ff6d00 con 0x558e83d41400 entity client.admin addr 2019-12-07T19:34:10.110+0000 7fe2e753b700 10 osd.0 32 ms_handle_authentication session 0x558e83ff6d00 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:10.110+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1746078500 conn(0x558e83d41400 0x558e8405a680 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=client.4406 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:10.110+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4406 172.21.15.131:0/1746078500 1 ==== command(tid 1: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558e8122c780 con 0x558e83d41400 2019-12-07T19:34:10.110+0000 7fe2d959f700 20 osd.0 32 OSD::ms_dispatch: command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:10.110+0000 7fe2d959f700 10 osd.0 32 do_waiters -- start 2019-12-07T19:34:10.110+0000 7fe2d959f700 10 osd.0 32 do_waiters -- finish 2019-12-07T19:34:10.110+0000 7fe2d959f700 20 osd.0 32 _dispatch 0x558e8122c780 command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:10.110+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/1746078500 -- command_reply(tid 1: 0 ) v1 -- 0x558e83c82a80 con 0x558e83d41400 2019-12-07T19:34:10.126+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.133+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4406 172.21.15.131:0/1746078500 2 ==== command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558e84198d20 con 0x558e83d41400 2019-12-07T19:34:10.133+0000 7fe2d959f700 20 osd.0 32 OSD::ms_dispatch: command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:10.133+0000 7fe2d959f700 10 osd.0 32 do_waiters -- start 2019-12-07T19:34:10.133+0000 7fe2d959f700 10 osd.0 32 do_waiters -- finish 2019-12-07T19:34:10.133+0000 7fe2d959f700 20 osd.0 32 _dispatch 0x558e84198d20 command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:10.134+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/1746078500 -- command_reply(tid 2: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558e83c828c0 con 0x558e83d41400 2019-12-07T19:34:10.136+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:10.176+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.226+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.243+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.5 172.21.15.109:0/36936 11 ==== osd_ping(ping e32 up_from 13 ping_stamp 2019-12-07T19:34:10.244089+0000/33.710412538s send_stamp 33.710412538s delta_ub 0.011990928s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83e68800 con 0x558e8181b800 2019-12-07T19:34:10.243+0000 7fe2e7d3c700 20 osd.0 32 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011392765s,0.011990928s]) 2019-12-07T19:34:10.243+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e32 up_from 13 ping_stamp 2019-12-07T19:34:10.244089+0000/33.710412538s send_stamp 33.699019773s delta_ub 18446744073.698158851s) v5 -- 0x558e845c2000 con 0x558e8181b800 2019-12-07T19:34:10.243+0000 7fe2e7d3c700 10 osd.0 32 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 31 -> 32 (as per caller) 2019-12-07T19:34:10.243+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.5 172.21.15.109:0/36936 11 ==== osd_ping(ping e32 up_from 13 ping_stamp 2019-12-07T19:34:10.244089+0000/33.710412538s send_stamp 33.710412538s delta_ub 0.011990928s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84053000 con 0x558e8182c800 2019-12-07T19:34:10.243+0000 7fe2e7d3c700 20 osd.0 32 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011338745s,0.011990928s]) 2019-12-07T19:34:10.243+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e32 up_from 13 ping_stamp 2019-12-07T19:34:10.244089+0000/33.710412538s send_stamp 33.699073793s delta_ub 18446744073.698212871s) v5 -- 0x558e845c2000 con 0x558e8182c800 2019-12-07T19:34:10.276+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.296+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1746078500 conn(0x558e83d41400 msgr2=0x558e8405a680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 61 2019-12-07T19:34:10.296+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1746078500 conn(0x558e83d41400 msgr2=0x558e8405a680 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:10.296+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1746078500 conn(0x558e83d41400 0x558e8405a680 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:10.296+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/1746078500 conn(0x558e83d41400 0x558e8405a680 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:10.296+0000 7fe2d959f700 2 osd.0 32 ms_handle_reset con 0x558e83d41400 session 0x558e83ff6d00 2019-12-07T19:34:10.326+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.376+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.426+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.456+0000 7fe2e2ca2700 10 osd.0 32 tick_without_osd_lock 2019-12-07T19:34:10.456+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fad0000/0x40000000/0x1680000000, data 0x4f9c39/0x52e000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:10.456+0000 7fe2e2ca2700 25 osd.0 32 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.655063+0000 last_rx_front 2019-12-07T19:34:08.655052+0000 2019-12-07T19:34:10.456+0000 7fe2e2ca2700 25 osd.0 32 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.655029+0000 last_rx_front 2019-12-07T19:34:08.655013+0000 2019-12-07T19:34:10.456+0000 7fe2e2ca2700 25 osd.0 32 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.654942+0000 last_rx_front 2019-12-07T19:34:08.654980+0000 2019-12-07T19:34:10.456+0000 7fe2e2ca2700 25 osd.0 32 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.655222+0000 last_rx_front 2019-12-07T19:34:08.655237+0000 2019-12-07T19:34:10.456+0000 7fe2e2ca2700 25 osd.0 32 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.655297+0000 last_rx_front 2019-12-07T19:34:08.655192+0000 2019-12-07T19:34:10.456+0000 7fe2e2ca2700 25 osd.0 32 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:08.654713+0000 last_rx_back 2019-12-07T19:34:08.655172+0000 last_rx_front 2019-12-07T19:34:08.655204+0000 2019-12-07T19:34:10.456+0000 7fe2e2ca2700 20 osd.0 32 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:10.456+0000 7fe2e2ca2700 10 osd.0 32 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:10.456+0000 7fe2e2ca2700 20 osd.0 32 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:10.456+0000 7fe2e2ca2700 10 osd.0 32 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:10.477+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.522+0000 7fe2d5281700 20 osd.0 32 reports for 0 queries 2019-12-07T19:34:10.522+0000 7fe2d5281700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- mgrreport(unknown.0 +0-0 packed 1198 daemon_metrics=2) v8 -- 0x558e83e15180 con 0x558e81748000 2019-12-07T19:34:10.522+0000 7fe2d5281700 20 bluestore(/var/lib/ceph/osd/ceph-0) pool_statfs pool 1 2019-12-07T19:34:10.522+0000 7fe2d5281700 10 bluestore(/var/lib/ceph/osd/ceph-0) pool_statfsstore_statfs(0x0/0x0/0x0, data 0x4d8000/0x4d8000, compress 0x0/0x0/0x0, omap 0x0, meta 0x0) 2019-12-07T19:34:10.522+0000 7fe2d5281700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:6800/36137,v1:172.21.15.131:6801/36137] -- pg_stats(1 pgs tid 0 v 0) v2 -- 0x558e83e15500 con 0x558e81748000 2019-12-07T19:34:10.527+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.574+0000 7fe2e4535700 10 osd.0 32 tick 2019-12-07T19:34:10.574+0000 7fe2e4535700 10 osd.0 32 do_waiters -- start 2019-12-07T19:34:10.574+0000 7fe2e4535700 10 osd.0 32 do_waiters -- finish 2019-12-07T19:34:10.574+0000 7fe2e4535700 20 osd.0 32 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:10.577+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 533081 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 130 ==== osd_map(32..33 src has 1..33) v4 ==== 1258+0+0 (unknown 0 0 0) 0x558e8178e240 con 0x558e818d7000 2019-12-07T19:34:10.612+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 131 ==== MOSDECSubOpRead(1.1s1 33/28 ECSubRead(tid=91, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 ==== 241+0+0 (unknown 0 0 0) 0x558e84242a00 con 0x558e818d7000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 osd.0 32 OSD::ms_dispatch: osd_map(32..33 src has 1..33) v4 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 osd.0 32 do_waiters -- start 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 osd.0 32 do_waiters -- finish 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 osd.0 32 _dispatch 0x558e8178e240 osd_map(32..33 src has 1..33) v4 2019-12-07T19:34:10.612+0000 7fe2e7d3c700 15 osd.0 32 enqueue_op 0x558e8477ba20 prio 127 cost 0 latency 0.000019 epoch 33 MOSDECSubOpRead(1.1s1 33/28 ECSubRead(tid=91, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:10.612+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 33/28 ECSubRead(tid=91, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e33) 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 33/28 ECSubRead(tid=91, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e33) queued 2019-12-07T19:34:10.612+0000 7fe2d4a80700 3 osd.0 32 handle_osd_map epochs [32,33], i have 32, src has [1,33] 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 osd.0 32 handle_osd_map got inc map for epoch 33 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGOpItem(op=MOSDECSubOpRead(1.1s1 33/28 ECSubRead(tid=91, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3) prio 127 cost 0 e33) pg 0x558e819e4000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3978e826:::osdmap.32:0# 0x0~0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1454 size 0x1454 (5204) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f61110 blob([0x2b6000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e841900e0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e847556c0 blob([!~1000,0x2b7000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558e84755650 sbid 0x0)) need 0x1000~454 cache has 0x[1000~454] 2019-12-07T19:34:10.612+0000 7fe2c7265700 10 osd.0 32 dequeue_op 0x558e8477ba20 prio 127 cost 0 latency 0.000054 MOSDECSubOpRead(1.1s1 33/28 ECSubRead(tid=91, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 pg pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3978e826:::osdmap.32:0# 0x0~1454 = 5204 2019-12-07T19:34:10.612+0000 7fe2c7265700 10 osd.0 32 maybe_share_map con 0x558e818d7000 v2:172.21.15.131:6812/37076 map epoch 31 -> 33 (as per caller) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 osd.0 32 add_map_bl 32 5204 bytes 2019-12-07T19:34:10.612+0000 7fe2c7265700 10 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] _handle_message: MOSDECSubOpRead(1.1s1 33/28 ECSubRead(tid=91, to_read={1:891f0d13:::smithi13140796-14:head=0,315392,0}, subchunks={1:891f0d13:::smithi13140796-14:head=[0,8]}, attrs_to_read=)) v3 2019-12-07T19:34:10.612+0000 7fe2c7265700 25 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] handle_sub_read case1: reading the complete chunk/shard. 2019-12-07T19:34:10.612+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x0~4d000 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~4d000 size 0x4d000 (315392) 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read defaulting to buffered read 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65730 blob([0x5c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c64070 loaded (sbid 0x2801 ref_map(0x5c000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c64e00 blob([0x5e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c657a0 loaded (sbid 0x2802 ref_map(0x5e000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83c65810 blob([0x60000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83c658f0 loaded (sbid 0x2803 ref_map(0x60000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9c00 blob([0x62000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9b90 loaded (sbid 0x2805 ref_map(0x62000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e9420 blob([0x64000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9650 loaded (sbid 0x2807 ref_map(0x64000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e95e0 blob([0x66000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e96c0 loaded (sbid 0x2809 ref_map(0x66000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e817e8850 blob([0x68000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9c70 loaded (sbid 0x280b ref_map(0x68000~2000=1)))) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194e5b0 blob([0x10d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fa40 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8194f880 blob([0x10f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9490 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81866bd0 blob([0x111000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a2f50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a2fc0 blob([0x113000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3030 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a30a0 blob([0x115000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3110 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3180 blob([0x156000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a31f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3260 blob([0x158000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a32d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3340 blob([0x15a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a33b0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3420 blob([0x15c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3490 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3500 blob([0x15e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3570 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a35e0 blob([0x160000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3650 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 osd.0 32 got_full_map 33, nothing requested 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a36c0 blob([0x162000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3730 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a37a0 blob([0x164000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3810 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3880 blob([0x166000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a38f0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3960 blob([0x168000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a39d0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3a40 blob([0x16a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3ab0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3b20 blob([0x16c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3b90 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3c00 blob([0x16e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3c70 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3ce0 blob([0x170000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3d50 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e841a3dc0 blob([0x172000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a3e30 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83d03f80 blob([0x296000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d5e0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d570 blob([0x298000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d500 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d490 blob([0x29a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d420 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d3b0 blob([0x29c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d340 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d2d0 blob([0x29e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d260 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d1f0 blob([0x2a0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d180 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 osd.0 32 handle_osd_map pg_num_history pg_num_history(e33 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps purged_snaps {33={}} 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d110 blob([0x2a2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423d0a0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 osd.0 32 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e33 [1,33] lci=[13,33]) 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423d030 blob([0x309000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cfc0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423cf50 blob([0x30b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423cee0 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e8423ce70 blob([0x30d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ce00 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e83d4db00 seq 25 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f95c70 blob([0x30f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a91570 sbid 0x0)) need 0x0~2000 cache has 0x[0~2000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e81a91500 blob([0x311000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8423d650 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:ca8b972f:::inc_osdmap.33:0# key 0x7f7fffffffffffffffca8b97'/!inc_osdmap.33!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:10.612+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) read 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 0x0~4d000 = 315392 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] handle_sub_read read request=315392 r=315392 len=315392 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:ca8b972f:::inc_osdmap.33:0# 0x0~24d 2019-12-07T19:34:10.612+0000 7fe2c7265700 10 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] get_hash_info: Getting attr on 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1156 2019-12-07T19:34:10.612+0000 7fe2c7265700 10 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] get_hash_info: not in cache 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:ca8b972f:::inc_osdmap.33:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:10.612+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) stat 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24d 2019-12-07T19:34:10.612+0000 7fe2c7265700 10 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] get_hash_info: found on disk, size 315392 2019-12-07T19:34:10.612+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83d4db00 1 blobs 2019-12-07T19:34:10.612+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) getattr 1.1s1_head 1#1:891f0d13:::smithi13140796-14:head# hinfo_key = 0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x2b8000~1000/1000,1000,0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x2b8000~1000] 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] handle_sub_read: Checking hash of 1:891f0d13:::smithi13140796-14:head 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83dd3030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84755500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83dd3030 blob([0x2b8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755500 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e83dd3030) get_ref 0x0~24d Blob(0x558e83dd3030 blob([0x2b8000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755500 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e83dd3030) get_ref init 0x1000, 1000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558e83dd3030 blob([0x2b8000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558e84755500 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e84755518 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x24d 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83b09788) dirty_range mark inline shard dirty 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:ca8b972f:::inc_osdmap.33:0# 0x0~24d = 0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3478e826:::osdmap.33:0# key 0x7f7fffffffffffffff3478e8'&!osdmap.33!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3478e826:::osdmap.33:0# 0x0~1438 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1157 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3478e826:::osdmap.33:0# 0x0~1438 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~438 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbc8 on front/back, now 0x0~1000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83d4db00 2 blobs 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x2b9000~2000/1000,2000,0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x2b9000~2000] 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84755420 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e847553b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84755420 blob([0x2b9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e847553b0 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e84755420) get_ref 0x0~1000 Blob(0x558e84755420 blob([0x2b9000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e847553b0 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e84755420) get_ref init 0x1000, 1000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e84755420 blob([0x2b9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e847553b0 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e847553c8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:10.612+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- MOSDECSubOpReadReply(1.1s0 32/28 ECSubReadReply(tid=91, attrs_read=0)) v2 -- 0x558e811dc280 con 0x558e818d7000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84755340 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e847552d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84755340 blob([!~1000,0x2ba000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e847552d0 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e84755340) get_ref 0x1000~438 Blob(0x558e84755340 blob([!~1000,0x2ba000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e847552d0 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e84755340) get_ref init 0x2000, 1000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~438: 0x1000~438 Blob(0x558e84755340 blob([!~1000,0x2ba000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558e847552d0 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e847552e8 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:10.612+0000 7fe2c7265700 10 osd.0 32 dequeue_op 0x558e8477ba20 finish 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1438 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8402f008) dirty_range mark inline shard dirty 2019-12-07T19:34:10.612+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3478e826:::osdmap.33:0# 0x0~1438 = 0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83d4db00 0 blobs 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83d4db00 0 blobs 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4db00 cost 23246 (4 ios * 4000 + 7246 bytes) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4db00 onodes 0x558e80251b80,0x558e817b9180,0x558e83b09680,0x558e8402ef00 shared_blobs 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83b09788) update #-1:ca8b972f:::inc_osdmap.33:0# 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e83b09788) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:ca8b972f:::inc_osdmap.33:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8402f008) update #-1:3478e826:::osdmap.33:0# 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8402f008) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3478e826:::osdmap.33:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4db00 allocated 0x[2b8000~3000] released 0x[] 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 freelist allocate 0x2b8000~3000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 20 freelist _xor first_key 0x280000 last_key 0x280000 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4db00 prepare 2019-12-07T19:34:10.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83d4db00 2019-12-07T19:34:10.612+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4db00 aio_wait 2019-12-07T19:34:10.612+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4db00 2019-12-07T19:34:10.612+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4db00 io_done 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:10.612+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b09680 had 1 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ef00 had 1 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:10.612+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:10.612+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:10.612+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:10.612+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x125131~226c to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:10.612+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x125131 0x125131~226c to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~100000 x_off 0xf5131 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x131 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x39d and padding block with 0xc63 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x12739d 2019-12-07T19:34:10.612+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:10.612+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:10.612+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000141372s (0.000005438s flush + 0.000135934s kv commit) 2019-12-07T19:34:10.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:10.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:10.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4db00> 2019-12-07T19:34:10.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:10.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4db00 kv_submitted 2019-12-07T19:34:10.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4db00 2019-12-07T19:34:10.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e83d4db00 osr 0x558e8107a500 2019-12-07T19:34:10.612+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e845eedc0) _discard drop 22 0x1b000~1000 2019-12-07T19:34:10.613+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e845eedc0) prepare_write seq 23 0x1b000~1000 crc 975caaf0 2019-12-07T19:34:10.613+0000 7fe2d7285700 10 osd.0 32 _committed_osd_maps 33..33 2019-12-07T19:34:10.613+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e845eedc0) _discard drop 22 0x2000~1000 2019-12-07T19:34:10.613+0000 7fe2d7285700 10 osd.0 32 advance to epoch 33 (<= last 33 <= newest_map 33) 2019-12-07T19:34:10.613+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e845eedc0) prepare_write seq 23 0x2000~1000 crc dd8e5cc9 2019-12-07T19:34:10.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:10.613+0000 7fe2d7285700 7 osd.0 33 consume_map version 33 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 33 identify_splits_and_merges 1.8s5 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 33 identify_splits_and_merges 1.9s3 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 33 identify_splits_and_merges 1.1s1 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 33 identify_splits_and_merges 1.as3 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 33 identify_splits_and_merges 1.2s0 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 33 identify_splits_and_merges 1.3s5 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 33 identify_splits_and_merges 1.bs1 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 33 identify_splits_and_merges 1.4s1 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 33 identify_splits_and_merges 1.5s4 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 33 identify_splits_and_merges 1.ds5 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 33 identify_splits_and_merges 1.es4 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 33 identify_splits_and_merges 1.7s3 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 33 identify_splits_and_merges 1.fs1 e32 to e33 pg_nums {18=16} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 33 prune_pg_created 2019-12-07T19:34:10.613+0000 7fe2d7285700 10 osd.0:0.consume_map 33 (was 32) 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 2019-12-07T19:34:10.613+0000 7fe2d7285700 10 osd.0:1.consume_map 33 (was 32) 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:10.613+0000 7fe2d7285700 10 osd.0:2.consume_map 33 (was 32) 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:10.613+0000 7fe2d7285700 10 osd.0:3.consume_map 33 (was 32) 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 2019-12-07T19:34:10.613+0000 7fe2d7285700 10 osd.0:4.consume_map 33 (was 32) 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:10.613+0000 7fe2d7285700 10 osd.0:5.consume_map 33 (was 32) 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:10.613+0000 7fe2d7285700 10 osd.0:6.consume_map 33 (was 32) 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:10.613+0000 7fe2d7285700 10 osd.0:7.consume_map 33 (was 32) 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:10.613+0000 7fe2d7285700 10 osd.0 33 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:10.613+0000 7fe2d7285700 15 osd.0 33 enqueue_peering_evt 1.8s5 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.613+0000 7fe2d7285700 15 osd.0 33 enqueue_peering_evt 1.9s3 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.613+0000 7fe2d7285700 15 osd.0 33 enqueue_peering_evt 1.1s1 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.613+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.613+0000 7fe2d7285700 15 osd.0 33 enqueue_peering_evt 1.as3 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.613+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558e841a4000 2019-12-07T19:34:10.613+0000 7fe2d7285700 15 osd.0 33 enqueue_peering_evt 1.2s0 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.613+0000 7fe2d7285700 15 osd.0 33 enqueue_peering_evt 1.3s5 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.613+0000 7fe2d7285700 15 osd.0 33 enqueue_peering_evt 1.bs1 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558e8193b000 2019-12-07T19:34:10.613+0000 7fe2d7285700 15 osd.0 33 enqueue_peering_evt 1.4s1 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 32 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.613+0000 7fe2d7285700 15 osd.0 33 enqueue_peering_evt 1.5s4 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2cba6e700 20 osd.0:0.update_pg_epoch 1.8s5 32 -> 33 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.613+0000 7fe2d7285700 15 osd.0 33 enqueue_peering_evt 1.ds5 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 32 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,4,5,2,3,0]/[1,4,5,2,3,0] -- 1/1 2019-12-07T19:34:10.613+0000 7fe2d7285700 15 osd.0 33 enqueue_peering_evt 1.es4 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 32 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.as3 32 -> 33 2019-12-07T19:34:10.613+0000 7fe2d7285700 15 osd.0 33 enqueue_peering_evt 1.7s3 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 33 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 32 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,4,2,0,5,3]/[1,4,2,0,5,3] -- 1/1 2019-12-07T19:34:10.613+0000 7fe2d7285700 15 osd.0 33 enqueue_peering_evt 1.fs1 epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) 2019-12-07T19:34:10.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 33 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.613+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.613+0000 7fe2d7285700 7 osd.0 33 activate_map version 33 2019-12-07T19:34:10.613+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558e81a98000 2019-12-07T19:34:10.613+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.8s5_head options 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 33 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.613+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2cba6e700 20 osd.0 pg_epoch: 33 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558e8192b000 2019-12-07T19:34:10.613+0000 7fe2cba6e700 20 osd.0 pg_epoch: 33 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.613+0000 7fe2cba6e700 20 osd.0 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 33 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 32 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.613+0000 7fe2c4a60700 20 osd.0:6.update_pg_epoch 1.es4 32 -> 33 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 33 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 33 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 32 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,3,4,5,0,2]/[1,3,4,5,0,2] -- 1/1 2019-12-07T19:34:10.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 33 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.613+0000 7fe2cba6e700 20 osd.0 pg_epoch: 33 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.613+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 33 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.9s3 32 -> 33 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 33 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 33 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 33 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.613+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:10.613+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 33 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.613+0000 7fe2c4a60700 20 osd.0 pg_epoch: 33 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558e81075000 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558e81231000 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558e81938000 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.613+0000 7fe2c4a60700 20 osd.0 pg_epoch: 33 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.613+0000 7fe2c4a60700 20 osd.0 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.613+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:34:10.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 33 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 33 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 pg_epoch: 33 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.2s0 32 -> 33 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 32 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 32 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.613+0000 7fe2c4a60700 20 osd.0 pg_epoch: 33 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0:3.update_pg_epoch 1.3s5 32 -> 33 2019-12-07T19:34:10.613+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 20 osd.0:4.update_pg_epoch 1.4s1 32 -> 33 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 pg_epoch: 33 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 32 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 33 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 32 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 32 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:10.613+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] state: Active advmap 2019-12-07T19:34:10.613+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 33 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 33 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 pg_epoch: 33 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 33 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 33 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.613+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:34:10.613+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.3s5_head options 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 33 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558e83d07000 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 pg_epoch: 33 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 pg_epoch: 33 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 32 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 33 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0:5.update_pg_epoch 1.5s4 32 -> 33 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] state: Active: handling ActMap 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] Active: kicking snap trim 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 33 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.613+0000 7fe2caa6c700 7 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] state: handle ActMap primary 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 32 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] handle_advance_map [2,1,3,4,0,5]/[2,1,3,4,0,5] -- 2/2 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 pg_epoch: 33 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558e819e4000 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 33 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.613+0000 7fe2caa6c700 15 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] publish_stats_to_osd 32:100 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 33 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 33 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.5s4_head options 2019-12-07T19:34:10.613+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 33 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558e83c78000 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.1s1 32 -> 33 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558e8122e000 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 pg_epoch: 33 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 32 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] handle_advance_map [3,0,2,4,5,1]/[3,0,2,4,5,1] -- 3/3 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 pg_epoch: 33 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 33 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 32 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 32 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 33 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.7s3 32 -> 33 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0:3.update_pg_epoch 1.bs1 32 -> 33 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 32 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 33 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 33 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 32 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:10.613+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 33 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 pg_epoch: 33 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 33 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 33 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.613+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 33 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 33 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 33 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 33 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.613+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs1_head options 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 pg_epoch: 33 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 pg_epoch: 33 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 pg_epoch: 33 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558e819e7000 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 33 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 pg_epoch: 33 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 33 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 33 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 pg_epoch: 33 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 pg_epoch: 33 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 33 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 32 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 pg_epoch: 33 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0:5.update_pg_epoch 1.ds5 32 -> 33 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 33 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 33 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 32 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 33 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 pg_epoch: 33 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) queued 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 33 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 33 epoch_requested: 33 NullEvt) prio 255 cost 10 e33) pg 0x558e81a9b000 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 33 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 33 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 pg_epoch: 33 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 33 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.613+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 32 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] handle_advance_map: 33 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.fs1 32 -> 33 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 osd.0 pg_epoch: 33 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 33 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.613+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 32 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] handle_advance_map [3,0,5,4,1,2]/[3,0,5,4,1,2] -- 3/3 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 33 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 pg_epoch: 33 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 33 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:10.613+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 pg_epoch: 33 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.613+0000 7fe2cc26f700 20 osd.0 33 agent_entry empty queue 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 33 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 pg_epoch: 33 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 33 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 pg_epoch: 33 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 33 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 33 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 33 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 pg_epoch: 33 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 pg_epoch: 33 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 33 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 osd.0 pg_epoch: 33 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 33 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 33 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 33 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 20 osd.0 33 advance_pg new pool opts old pool opts 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 33 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] handle_activate_map: 33 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 33 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 33 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 33 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 33 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 33 epoch_requested: 33 NullEvt 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:10.613+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:10.618+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 246 ==== osd_map(32..33 src has 1..33) v4 ==== 1258+0+0 (unknown 0 0 0) 0x558e8178fb00 con 0x558e818d7c00 2019-12-07T19:34:10.618+0000 7fe2d4a80700 20 osd.0 33 OSD::ms_dispatch: osd_map(32..33 src has 1..33) v4 2019-12-07T19:34:10.618+0000 7fe2d4a80700 10 osd.0 33 do_waiters -- start 2019-12-07T19:34:10.618+0000 7fe2d4a80700 10 osd.0 33 do_waiters -- finish 2019-12-07T19:34:10.618+0000 7fe2d4a80700 20 osd.0 33 _dispatch 0x558e8178fb00 osd_map(32..33 src has 1..33) v4 2019-12-07T19:34:10.618+0000 7fe2d4a80700 3 osd.0 33 handle_osd_map epochs [32,33], i have 33, src has [1,33] 2019-12-07T19:34:10.618+0000 7fe2d4a80700 10 osd.0 33 no new maps here, dropping 2019-12-07T19:34:10.618+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 247 ==== MOSDECSubOpWrite(1.9s3 33/28 ECSubWrite(tid=154, reqid=client.4540.0:270, at_version=33'15, trim_to=0'0, roll_forward_to=29'13)) v2 ==== 372651+0+0 (unknown 0 0 0) 0x558e84041800 con 0x558e818d7c00 2019-12-07T19:34:10.619+0000 7fe2e7d3c700 15 osd.0 33 enqueue_op 0x558e81943e40 prio 127 cost 0 latency 0.000042 epoch 33 MOSDECSubOpWrite(1.9s3 33/28 ECSubWrite(tid=154, reqid=client.4540.0:270, at_version=33'15, trim_to=0'0, roll_forward_to=29'13)) v2 2019-12-07T19:34:10.619+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 33/28 ECSubWrite(tid=154, reqid=client.4540.0:270, at_version=33'15, trim_to=0'0, roll_forward_to=29'13)) v2) prio 127 cost 0 e33) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 33/28 ECSubWrite(tid=154, reqid=client.4540.0:270, at_version=33'15, trim_to=0'0, roll_forward_to=29'13)) v2) prio 127 cost 0 e33) queued 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 33/28 ECSubWrite(tid=154, reqid=client.4540.0:270, at_version=33'15, trim_to=0'0, roll_forward_to=29'13)) v2) prio 127 cost 0 e33) pg 0x558e8192b000 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 osd.0 33 dequeue_op 0x558e81943e40 prio 127 cost 0 latency 0.000085 MOSDECSubOpWrite(1.9s3 33/28 ECSubWrite(tid=154, reqid=client.4540.0:270, at_version=33'15, trim_to=0'0, roll_forward_to=29'13)) v2 pg pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 osd.0 33 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 31 -> 33 (as per caller) 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] _handle_message: MOSDECSubOpWrite(1.9s3 33/28 ECSubWrite(tid=154, reqid=client.4540.0:270, at_version=33'15, trim_to=0'0, roll_forward_to=29'13)) v2 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 snap_mapper.add_oid 1:9b8adabe:::smithi13140796-6:a a 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 snap_mapper.get_snaps 1:9b8adabe:::smithi13140796-6:a got.empty() 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 snap_mapper.set_snaps 1:9b8adabe:::smithi13140796-6:a a 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 snap_mapper.set_snaps set OBJ_.3_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 snap_mapper.add_oid set SNA_1_000000000000000A_.3_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 29'13 (0'0,29'13] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] append_log log((0'0,29'13], crt=29'13) [33'14 (21'5) clone 1:9b8adabe:::smithi13140796-6:a by unknown.0.0:0 2019-12-07T19:34:01.670030+0000 0 snaps [a] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,33'15 (21'5) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:270 0.000000 0 ObjectCleanRegions clean_offsets: [1261568~18446744073708290047], clean_omap: 1, new_object: 0] 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 33'14 (0'0,33'14] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'13 crt=29'13 lcod 29'13 active mbc={} ps=[4~1,6~2]] add_log_entry 33'14 (21'5) clone 1:9b8adabe:::smithi13140796-6:a by unknown.0.0:0 2019-12-07T19:34:01.670030+0000 0 snaps [a] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'13 crt=29'13 lcod 29'13 active mbc={} ps=[4~1,6~2]] add_log_entry 33'15 (21'5) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:270 0.000000 0 ObjectCleanRegions clean_offsets: [1261568~18446744073708290047], clean_omap: 1, new_object: 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'13 crt=29'13 lcod 29'13 active mbc={} ps=[4~1,6~2]] append_log approx pg log length = 15 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'13 crt=29'13 lcod 29'13 active mbc={} ps=[4~1,6~2]] append_log transaction_applied = 1 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:10.619+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 33'14, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83eb8000 seq 35 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) get_onode oid 3#1:9b8adabe:::smithi13140796-6:a# key 0x8380000000000000019b8adabe21736d'ithi13140796-6!='0x000000000000000affffffffffffffff'o' 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) r -2 v.len 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _clone 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# -> 3#1:9b8adabe:::smithi13140796-6:a# 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1158 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.onode(0x558e811dd900).flush flush done 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:a# 0x0 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_clone_range 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# -> 3#1:9b8adabe:::smithi13140796-6:a# 0x0~6000 -> 0x0~6000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) dup src 0x0~2000: 0x0~2000 Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10280 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) make_blob_shared Blob(0x558e8194e2a0 blob([0x3f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) make_blob_shared now Blob(0x558e8194e2a0 blob([0x3f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 loaded (sbid 0x2828 ref_map(0x3f000~2000=1)))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) dup new Blob(0x558e84755b20 blob([0x3f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e818667e0 loaded (sbid 0x2828 ref_map(0x3f000~2000=2)))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84755b20) get_ref 0x0~2000 Blob(0x558e84755b20 blob([0x3f000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e818667e0 loaded (sbid 0x2828 ref_map(0x3f000~2000=2)))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84755b20) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) dup dst 0x0~2000: 0x0~2000 Blob(0x558e84755b20 blob([0x3f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 loaded (sbid 0x2828 ref_map(0x3f000~2000=2)))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) dup src 0x2000~2000: 0x0~2000 Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10281 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) make_blob_shared Blob(0x558e8194e310 blob([0x41000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) make_blob_shared now Blob(0x558e8194e310 blob([0x41000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 loaded (sbid 0x2829 ref_map(0x41000~2000=1)))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) dup new Blob(0x558e84755b90 blob([0x41000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e1c0 loaded (sbid 0x2829 ref_map(0x41000~2000=2)))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84755b90) get_ref 0x0~2000 Blob(0x558e84755b90 blob([0x41000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e1c0 loaded (sbid 0x2829 ref_map(0x41000~2000=2)))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84755b90) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) dup dst 0x2000~2000: 0x0~2000 Blob(0x558e84755b90 blob([0x41000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 loaded (sbid 0x2829 ref_map(0x41000~2000=2)))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) dup src 0x4000~2000: 0x0~2000 Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10282 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) make_blob_shared Blob(0x558e81866460 blob([0x43000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) make_blob_shared now Blob(0x558e81866460 blob([0x43000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 loaded (sbid 0x282a ref_map(0x43000~2000=1)))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) dup new Blob(0x558e83ac4ee0 blob([0x43000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81867960 loaded (sbid 0x282a ref_map(0x43000~2000=2)))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e83ac4ee0) get_ref 0x0~2000 Blob(0x558e83ac4ee0 blob([0x43000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81867960 loaded (sbid 0x282a ref_map(0x43000~2000=2)))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e83ac4ee0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) dup dst 0x4000~2000: 0x0~2000 Blob(0x558e83ac4ee0 blob([0x43000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 loaded (sbid 0x282a ref_map(0x43000~2000=2)))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) dirty_range mark inline shard dirty 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e8402e608) dirty_range mark inline shard dirty 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _clone 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# -> 3#1:9b8adabe:::smithi13140796-6:a# = 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rmattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:a# snapset 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rmattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:a# snapset = 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:a# 1 keys 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:a# 1 keys = 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:a# hinfo_key (42 bytes) 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:a# hinfo_key (42 bytes) = 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) get_onode oid 3#1:9b8adabe:::smithi13140796-6:head#f key 0x8380000000000000019b8adabe21736d'ithi13140796-6!='0xfffffffffffffffe000000000000000f'o' 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) r -2 v.len 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# -> 3#1:9b8adabe:::smithi13140796-6:head#f 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# -> 3#1:9b8adabe:::smithi13140796-6:head#f = 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1159 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# = 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 23 keys 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 23 keys = 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x0~4d000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 3#1:9b8adabe:::smithi13140796-6:head# 0x0~4d000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~4d000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83eb8000 39 blobs 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 fbmap_alloc 0x558e80233300 allocate 0x4d000/1000,4d000,0 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x2bb000~4d000/1000,4d000,0 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x2bb000~4d000] 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84755ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84755a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84755ab0 blob([0x2bb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755a40 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84755ab0) get_ref 0x0~2000 Blob(0x558e84755ab0 blob([0x2bb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755a40 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84755ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e84755ab0 blob([0x2bb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84755a40 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84755a58 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84755960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e847558f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84755960 blob([0x2bd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e847558f0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84755960) get_ref 0x0~2000 Blob(0x558e84755960 blob([0x2bd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e847558f0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84755960) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e84755960 blob([0x2bd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e847558f0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84755908 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84755880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84755810 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84755880 blob([0x2bf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755810 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84755880) get_ref 0x0~2000 Blob(0x558e84755880 blob([0x2bf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755810 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84755880) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e84755880 blob([0x2bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84755810 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84755828 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e847557a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84755730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e847557a0 blob([0x2c1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755730 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e847557a0) get_ref 0x0~2000 Blob(0x558e847557a0 blob([0x2c1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755730 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e847557a0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e847557a0 blob([0x2c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84755730 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84755748 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84754c40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84754a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84754c40 blob([0x2c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84754a10 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84754c40) get_ref 0x0~2000 Blob(0x558e84754c40 blob([0x2c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84754a10 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84754c40) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e84754c40 blob([0x2c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84754a10 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84754a28 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84754850 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e847547e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84754850 blob([0x2c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e847547e0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84754850) get_ref 0x0~2000 Blob(0x558e84754850 blob([0x2c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e847547e0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84754850) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e84754850 blob([0x2c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e847547e0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e847547f8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84754930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e847549a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84754930 blob([0x2c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e847549a0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84754930) get_ref 0x0~2000 Blob(0x558e84754930 blob([0x2c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e847549a0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84754930) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e84754930 blob([0x2c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e847549a0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e847549b8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84755f80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84755f10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84755f80 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755f10 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84755f80) get_ref 0x0~2000 Blob(0x558e84755f80 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755f10 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84755f80) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e84755f80 blob([0x2c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84755f10 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84755f28 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84755ea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84755e30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84755ea0 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755e30 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84755ea0) get_ref 0x0~2000 Blob(0x558e84755ea0 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755e30 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84755ea0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e84755ea0 blob([0x2cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84755e30 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84755e48 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8472f960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f95d50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8472f960 blob([0x2cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f95d50 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8472f960) get_ref 0x0~2000 Blob(0x558e8472f960 blob([0x2cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f95d50 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8472f960) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e8472f960 blob([0x2cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f95d50 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83f95d68 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6d5e0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6d570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6d5e0 blob([0x2cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6d570 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e83e6d5e0) get_ref 0x0~2000 Blob(0x558e83e6d5e0 blob([0x2cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6d570 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e83e6d5e0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83e6d5e0 blob([0x2cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6d570 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83e6d588 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423cbd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423cbd0 blob([0x2d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645180 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8423cbd0) get_ref 0x0~2000 Blob(0x558e8423cbd0 blob([0x2d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645180 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8423cbd0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e8423cbd0 blob([0x2d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645180 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84645198 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846451f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846457a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846451f0 blob([0x2d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846457a0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e846451f0) get_ref 0x0~2000 Blob(0x558e846451f0 blob([0x2d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846457a0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e846451f0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e846451f0 blob([0x2d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846457a0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e846457b8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84548e00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846450a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84548e00 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846450a0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84548e00) get_ref 0x0~2000 Blob(0x558e84548e00 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846450a0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84548e00) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e84548e00 blob([0x2d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846450a0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e846450b8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84645110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84645110 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645730 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84645110) get_ref 0x0~2000 Blob(0x558e84645110 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645730 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84645110) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e84645110 blob([0x2d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645730 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84645748 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84548540 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84644fc0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84548540 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644fc0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84548540) get_ref 0x0~2000 Blob(0x558e84548540 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644fc0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84548540) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e84548540 blob([0x2d9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644fc0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84644fd8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84645030 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846456c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84645030 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846456c0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84645030) get_ref 0x0~2000 Blob(0x558e84645030 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846456c0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84645030) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e84645030 blob([0x2db000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846456c0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e846456d8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84548bd0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84644ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84548bd0 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644ee0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84548bd0) get_ref 0x0~2000 Blob(0x558e84548bd0 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644ee0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84548bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e84548bd0 blob([0x2dd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644ee0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84644ef8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84644f50 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84644f50 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645650 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84644f50) get_ref 0x0~2000 Blob(0x558e84644f50 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645650 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84644f50) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e84644f50 blob([0x2df000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645650 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84645668 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a620 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84644e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a620 blob([0x2e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644e00 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8456a620) get_ref 0x0~2000 Blob(0x558e8456a620 blob([0x2e1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644e00 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8456a620) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x26000~2000: 0x0~2000 Blob(0x558e8456a620 blob([0x2e1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644e00 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84644e18 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84644e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e846455e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84644e70 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846455e0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84644e70) get_ref 0x0~2000 Blob(0x558e84644e70 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e846455e0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84644e70) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x28000~2000: 0x0~2000 Blob(0x558e84644e70 blob([0x2e3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e846455e0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e846455f8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e845491f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84644d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e845491f0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644d20 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e845491f0) get_ref 0x0~2000 Blob(0x558e845491f0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84644d20 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e845491f0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2a000~2000: 0x0~2000 Blob(0x558e845491f0 blob([0x2e5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84644d20 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84644d38 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84644d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645570 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84644d90 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645570 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84644d90) get_ref 0x0~2000 Blob(0x558e84644d90 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645570 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84644d90) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2c000~2000: 0x0~2000 Blob(0x558e84644d90 blob([0x2e7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645570 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84645588 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84548d20 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84548d20 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b500 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84548d20) get_ref 0x0~2000 Blob(0x558e84548d20 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b500 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84548d20) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2e000~2000: 0x0~2000 Blob(0x558e84548d20 blob([0x2e9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5b500 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83f5b518 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84644cb0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84644cb0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645500 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84644cb0) get_ref 0x0~2000 Blob(0x558e84644cb0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645500 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e84644cb0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x30000~2000: 0x0~2000 Blob(0x558e84644cb0 blob([0x2eb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645500 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84645518 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e845484d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b3b0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e845484d0 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b3b0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e845484d0) get_ref 0x0~2000 Blob(0x558e845484d0 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f5b3b0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e845484d0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x32000~2000: 0x0~2000 Blob(0x558e845484d0 blob([0x2ed000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f5b3b0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83f5b3c8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f5b490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84645490 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f5b490 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645490 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e83f5b490) get_ref 0x0~2000 Blob(0x558e83f5b490 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84645490 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e83f5b490) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x34000~2000: 0x0~2000 Blob(0x558e83f5b490 blob([0x2ef000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84645490 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e846454a8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f61960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8456a000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f61960 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a000 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e83f61960) get_ref 0x0~2000 Blob(0x558e83f61960 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8456a000 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e83f61960) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x36000~2000: 0x0~2000 Blob(0x558e83f61960 blob([0x2f1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8456a000 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8456a018 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456aaf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84549340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456aaf0 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84549340 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8456aaf0) get_ref 0x0~2000 Blob(0x558e8456aaf0 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84549340 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8456aaf0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x38000~2000: 0x0~2000 Blob(0x558e8456aaf0 blob([0x2f3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84549340 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84549358 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464dea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464df10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464dea0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464df10 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8464dea0) get_ref 0x0~2000 Blob(0x558e8464dea0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464df10 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8464dea0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3a000~2000: 0x0~2000 Blob(0x558e8464dea0 blob([0x2f5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464df10 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8464df28 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456aa80 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e845492d0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456aa80 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e845492d0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8456aa80) get_ref 0x0~2000 Blob(0x558e8456aa80 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e845492d0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8456aa80) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3c000~2000: 0x0~2000 Blob(0x558e8456aa80 blob([0x2f7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e845492d0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e845492e8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464ddc0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464de30 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464ddc0 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464de30 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8464ddc0) get_ref 0x0~2000 Blob(0x558e8464ddc0 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464de30 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8464ddc0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x3e000~2000: 0x0~2000 Blob(0x558e8464ddc0 blob([0x2f9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464de30 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8464de48 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456aa10 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84549260 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456aa10 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84549260 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8456aa10) get_ref 0x0~2000 Blob(0x558e8456aa10 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84549260 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8456aa10) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x40000~2000: 0x0~2000 Blob(0x558e8456aa10 blob([0x2fb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84549260 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e84549278 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464dce0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464dd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464dce0 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464dd50 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8464dce0) get_ref 0x0~2000 Blob(0x558e8464dce0 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464dd50 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8464dce0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x42000~2000: 0x0~2000 Blob(0x558e8464dce0 blob([0x2fd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464dd50 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8464dd68 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a9a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f616c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a9a0 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f616c0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8456a9a0) get_ref 0x0~2000 Blob(0x558e8456a9a0 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f616c0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8456a9a0) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x44000~2000: 0x0~2000 Blob(0x558e8456a9a0 blob([0x2ff000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f616c0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83f616d8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8464dc00 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464dc70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8464dc00 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464dc70 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8464dc00) get_ref 0x0~2000 Blob(0x558e8464dc00 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464dc70 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8464dc00) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x46000~2000: 0x0~2000 Blob(0x558e8464dc00 blob([0x301000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464dc70 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8464dc88 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a930 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f618f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a930 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f618f0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8456a930) get_ref 0x0~2000 Blob(0x558e8456a930 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f618f0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8456a930) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x48000~2000: 0x0~2000 Blob(0x558e8456a930 blob([0x303000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83f618f0 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83f61908 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83ed4070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8464db90 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83ed4070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464db90 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e83ed4070) get_ref 0x0~2000 Blob(0x558e83ed4070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8464db90 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e83ed4070) get_ref init 0x2000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4a000~2000: 0x0~2000 Blob(0x558e83ed4070 blob([0x305000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8464db90 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e8464dba8 in 0x558e80219a40) _discard 0x0~2000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8456a8c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f61880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8456a8c0 blob([0x307000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61880 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8456a8c0) get_ref 0x0~1000 Blob(0x558e8456a8c0 blob([0x307000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f61880 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.blob(0x558e8456a8c0) get_ref init 0x1000, 1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4c000~1000: 0x0~1000 Blob(0x558e8456a8c0 blob([0x307000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f61880 sbid 0x0)) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.BufferSpace(0x558e83f61898 in 0x558e80219a40) _discard 0x0~1000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x4d000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e83c72d88) dirty_range mark inline shard dirty 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 0x0~4d000 = 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.3_0000000000000001.9D15B5D7.a.smithi13140796-6..' <- OBJ_.3_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000A_.3_0000000000000001.9D15B5D7.a.smithi13140796-6..' <- SNA_1_000000000000000A_.3_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000033.00000000000000000014' <- 0000000033.00000000000000000014 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000033.00000000000000000015' <- 0000000033.00000000000000000015 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83eb8000 cost 533693 (40 ios * 4000 + 373693 bytes) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83eb8000 onodes 0x558e811dd900,0x558e83c72c80,0x558e8402e500 shared_blobs 0x558e818667e0,0x558e81867960,0x558e8194e1c0 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) update 3#1:9b8adabe:::smithi13140796-6:head#f 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) update inline shard 84 bytes from 3 extents 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9b8adabe:::smithi13140796-6:head#f is 573 (483 bytes onode + 2 bytes spanning blobs + 88 bytes inline extents) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e83c72d88) update 3#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e83c72d88) update inline shard 741 bytes from 39 extents 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9b8adabe:::smithi13140796-6:head# is 55192 (54445 bytes onode + 2 bytes spanning blobs + 745 bytes inline extents) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e8402e608) update 3#1:9b8adabe:::smithi13140796-6:a# 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore.extentmap(0x558e8402e608) update inline shard 84 bytes from 3 extents 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9b8adabe:::smithi13140796-6:a# is 500 (410 bytes onode + 2 bytes spanning blobs + 88 bytes inline extents) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2828 is 11 SharedBlob(0x558e818667e0 loaded (sbid 0x2828 ref_map(0x3f000~2000=2))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x282a is 11 SharedBlob(0x558e81867960 loaded (sbid 0x282a ref_map(0x43000~2000=2))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2829 is 11 SharedBlob(0x558e8194e1c0 loaded (sbid 0x2829 ref_map(0x41000~2000=2))) 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83eb8000 allocated 0x[2bb000~4d000] released 0x[] 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 freelist allocate 0x2bb000~4d000 2019-12-07T19:34:10.619+0000 7fe2c7265700 20 freelist _xor first_key 0x280000 last_key 0x300000 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8000 prepare 2019-12-07T19:34:10.619+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83eb8000 2019-12-07T19:34:10.620+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=29'13 crt=29'13 lcod 29'13 active mbc={} ps=[4~1,6~2]] op_applied version 33'15 2019-12-07T19:34:10.620+0000 7fe2c7265700 10 osd.0 33 dequeue_op 0x558e81943e40 finish 2019-12-07T19:34:10.620+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.620+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.620+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8000 aio_wait 2019-12-07T19:34:10.620+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83eb8000 2019-12-07T19:34:10.620+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8000 io_done 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3171 < min_flush_size 524288 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 35939 < min_flush_size 524288 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 58870 < min_flush_size 524288 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dd900 had 1 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72c80 had 1 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402e500 had 1 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 58893 < min_flush_size 524288 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 58893 < min_flush_size 524288 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x12739d~e60d to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x12739d 0x12739d~e60d to file(ino 16 size 0x130000 mtime 2019-12-07T19:34:09.762457+0000 allocated 130000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~100000]) 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _allocate len 0x59aa from 0 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _allocate len 0x59aa from 1 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd260000 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd260000~10000/10000,10000,acd260000 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x140000 to include allocated 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 38 (was clean) 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x140000 mtime 2019-12-07T19:34:10.621116+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~110000 x_off 0xf739d 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x39d 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x9aa and padding block with 0x656 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x140000 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1359aa 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (38) on file(ino 16 size 0x140000 mtime 2019-12-07T19:34:10.621116+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]), flushing log 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x140000 mtime 2019-12-07T19:34:10.621116+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 38 len 0x49 crc 0x404d6974) 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x25000~1000 to file(ino 1 size 0x25000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x25000 0x25000~1000 to file(ino 1 size 0x25000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x26000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x25000 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x26000 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:34:10.620+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000026 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 38 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x140000 mtime 2019-12-07T19:34:10.621116+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000461677s (0.000005310s flush + 0.000456367s kv commit) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:10.620+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83eb8000> 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:10.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8000 kv_submitted 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83eb8000 2019-12-07T19:34:10.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8000 finishing 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83eb8000 onodes 0x558e811dd900,0x558e83c72c80,0x558e8402e500 2019-12-07T19:34:10.620+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6d588 in 0x558e80219a40) _finish_write discard buffer(0x558e84758750 space 0x558e83e6d588 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5b3c8 in 0x558e80219a40) _finish_write discard buffer(0x558e84759b90 space 0x558e83f5b3c8 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f5b518 in 0x558e80219a40) _finish_write discard buffer(0x558e84759a70 space 0x558e83f5b518 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f616d8 in 0x558e80219a40) _finish_write discard buffer(0x558e847b0090 space 0x558e83f616d8 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f61898 in 0x558e80219a40) _finish_write discard buffer(0x558e847b02d0 space 0x558e83f61898 0x0~1000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f61908 in 0x558e80219a40) _finish_write discard buffer(0x558e847b01b0 space 0x558e83f61908 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f95d68 in 0x558e80219a40) _finish_write discard buffer(0x558e847586c0 space 0x558e83f95d68 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84549278 in 0x558e80219a40) _finish_write discard buffer(0x558e840323f0 space 0x558e84549278 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e845492e8 in 0x558e80219a40) _finish_write discard buffer(0x558e84759e60 space 0x558e845492e8 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84549358 in 0x558e80219a40) _finish_write discard buffer(0x558e84759d40 space 0x558e84549358 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8456a018 in 0x558e80219a40) _finish_write discard buffer(0x558e84759cb0 space 0x558e8456a018 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84644d38 in 0x558e80219a40) _finish_write discard buffer(0x558e84759950 space 0x558e84644d38 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84644e18 in 0x558e80219a40) _finish_write discard buffer(0x558e84759830 space 0x558e84644e18 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84644ef8 in 0x558e80219a40) _finish_write discard buffer(0x558e84758b40 space 0x558e84644ef8 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84644fd8 in 0x558e80219a40) _finish_write discard buffer(0x558e84758a20 space 0x558e84644fd8 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846450b8 in 0x558e80219a40) _finish_write discard buffer(0x558e84758900 space 0x558e846450b8 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645198 in 0x558e80219a40) _finish_write discard buffer(0x558e847587e0 space 0x558e84645198 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846454a8 in 0x558e80219a40) _finish_write discard buffer(0x558e84759c20 space 0x558e846454a8 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645518 in 0x558e80219a40) _finish_write discard buffer(0x558e84759b00 space 0x558e84645518 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645588 in 0x558e80219a40) _finish_write discard buffer(0x558e847599e0 space 0x558e84645588 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846455f8 in 0x558e80219a40) _finish_write discard buffer(0x558e847598c0 space 0x558e846455f8 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645668 in 0x558e80219a40) _finish_write discard buffer(0x558e847597a0 space 0x558e84645668 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846456d8 in 0x558e80219a40) _finish_write discard buffer(0x558e84758ab0 space 0x558e846456d8 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84645748 in 0x558e80219a40) _finish_write discard buffer(0x558e84758990 space 0x558e84645748 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e846457b8 in 0x558e80219a40) _finish_write discard buffer(0x558e84758870 space 0x558e846457b8 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464dba8 in 0x558e80219a40) _finish_write discard buffer(0x558e847b0240 space 0x558e8464dba8 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464dc88 in 0x558e80219a40) _finish_write discard buffer(0x558e847b0120 space 0x558e8464dc88 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464dd68 in 0x558e80219a40) _finish_write discard buffer(0x558e847b0000 space 0x558e8464dd68 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464de48 in 0x558e80219a40) _finish_write discard buffer(0x558e84759ef0 space 0x558e8464de48 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8464df28 in 0x558e80219a40) _finish_write discard buffer(0x558e84759dd0 space 0x558e8464df28 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e847547f8 in 0x558e80219a40) _finish_write discard buffer(0x558e84758480 space 0x558e847547f8 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e847549b8 in 0x558e80219a40) _finish_write discard buffer(0x558e84758510 space 0x558e847549b8 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84754a28 in 0x558e80219a40) _finish_write discard buffer(0x558e847583f0 space 0x558e84754a28 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84755748 in 0x558e80219a40) _finish_write discard buffer(0x558e84758360 space 0x558e84755748 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84755828 in 0x558e80219a40) _finish_write discard buffer(0x558e847582d0 space 0x558e84755828 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84755908 in 0x558e80219a40) _finish_write discard buffer(0x558e84758240 space 0x558e84755908 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84755a58 in 0x558e80219a40) _finish_write discard buffer(0x558e847581b0 space 0x558e84755a58 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84755e48 in 0x558e80219a40) _finish_write discard buffer(0x558e84758630 space 0x558e84755e48 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84755f28 in 0x558e80219a40) _finish_write discard buffer(0x558e847585a0 space 0x558e84755f28 0x0~2000 writing nocache) 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83eb8000 done 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:10.620+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83eb8000 [] 2019-12-07T19:34:10.620+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:10.620+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:10.620+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 33/28 ECSubWriteReply(tid=154, last_complete=33'15, committed=1, applied=1)) v2 -- 0x558e83fee600 con 0x558e818d7c00 2019-12-07T19:34:10.620+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.623+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 248 ==== MOSDECSubOpWrite(1.9s3 33/28 ECSubWrite(tid=155, reqid=client.4540.0:270, at_version=0'0, trim_to=0'0, roll_forward_to=33'15)) v2 ==== 909+0+0 (unknown 0 0 0) 0x558e819cf800 con 0x558e818d7c00 2019-12-07T19:34:10.623+0000 7fe2e7d3c700 15 osd.0 33 enqueue_op 0x558e81942b00 prio 127 cost 0 latency 0.000026 epoch 33 MOSDECSubOpWrite(1.9s3 33/28 ECSubWrite(tid=155, reqid=client.4540.0:270, at_version=0'0, trim_to=0'0, roll_forward_to=33'15)) v2 2019-12-07T19:34:10.623+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 33/28 ECSubWrite(tid=155, reqid=client.4540.0:270, at_version=0'0, trim_to=0'0, roll_forward_to=33'15)) v2) prio 127 cost 0 e33) 2019-12-07T19:34:10.623+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:10.623+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 33/28 ECSubWrite(tid=155, reqid=client.4540.0:270, at_version=0'0, trim_to=0'0, roll_forward_to=33'15)) v2) prio 127 cost 0 e33) queued 2019-12-07T19:34:10.623+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:10.623+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 33/28 ECSubWrite(tid=155, reqid=client.4540.0:270, at_version=0'0, trim_to=0'0, roll_forward_to=33'15)) v2) prio 127 cost 0 e33) pg 0x558e8192b000 2019-12-07T19:34:10.623+0000 7fe2c7265700 10 osd.0 33 dequeue_op 0x558e81942b00 prio 127 cost 0 latency 0.000059 MOSDECSubOpWrite(1.9s3 33/28 ECSubWrite(tid=155, reqid=client.4540.0:270, at_version=0'0, trim_to=0'0, roll_forward_to=33'15)) v2 pg pg[1.9s3( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] 2019-12-07T19:34:10.623+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] _handle_message: MOSDECSubOpWrite(1.9s3 33/28 ECSubWrite(tid=155, reqid=client.4540.0:270, at_version=0'0, trim_to=0'0, roll_forward_to=33'15)) v2 2019-12-07T19:34:10.623+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=29'13 active mbc={} ps=[4~1,6~2]] append_log log((0'0,33'15], crt=29'13) [] 2019-12-07T19:34:10.623+0000 7fe2c7265700 20 osd.0 pg_epoch: 33 pg[1.9s3( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] rollforward: entry=33'14 (21'5) clone 1:9b8adabe:::smithi13140796-6:a by unknown.0.0:0 2019-12-07T19:34:01.670030+0000 0 snaps [a] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 osd.0 pg_epoch: 33 pg[1.9s3( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] rollforward: entry=33'15 (21'5) modify 1:9b8adabe:::smithi13140796-6:head by client.4540.0:270 0.000000 0 ObjectCleanRegions clean_offsets: [1261568~18446744073708290047], clean_omap: 1, new_object: 0 2019-12-07T19:34:10.624+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] append_log approx pg log length = 15 2019-12-07T19:34:10.624+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] append_log transaction_applied = 1 2019-12-07T19:34:10.624+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:10.624+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:10.624+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83eb8300 seq 36 2019-12-07T19:34:10.624+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head#f onode 0x558e811dd900 txc 0x558e83eb8300 2019-12-07T19:34:10.624+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head#f 0x0 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore.blob(0x558e8194e2a0) put_ref 0x0~2000 Blob(0x558e8194e2a0 blob([0x3f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 loaded (sbid 0x2828 ref_map(0x3f000~2000=2)))) 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore.blob(0x558e8194e310) put_ref 0x0~2000 Blob(0x558e8194e310 blob([0x41000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 loaded (sbid 0x2829 ref_map(0x41000~2000=2)))) 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore.blob(0x558e81866460) put_ref 0x0~2000 Blob(0x558e81866460 blob([0x43000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 loaded (sbid 0x282a ref_map(0x43000~2000=2)))) 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore.extentmap(0x558e811dda08) dirty_range mark inline shard dirty 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e8194e2a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e818667e0 loaded (sbid 0x2828 ref_map(0x3f000~2000=2)))) 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x3f000~2000] 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e818667e0 loaded (sbid 0x2828 ref_map(0x3f000~2000=1))) 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e8194e310 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8194e1c0 loaded (sbid 0x2829 ref_map(0x41000~2000=2)))) 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x41000~2000] 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e8194e1c0 loaded (sbid 0x2829 ref_map(0x41000~2000=1))) 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e81866460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e81867960 loaded (sbid 0x282a ref_map(0x43000~2000=2)))) 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x43000~2000] 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e81867960 loaded (sbid 0x282a ref_map(0x43000~2000=1))) 2019-12-07T19:34:10.624+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove gen and maybe_unshared_blobs 0x558e818667e0,0x558e81867960,0x558e8194e1c0 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove checking for unshareable blobs on 0x558e83c72c80 3#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:10.624+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head#f = 0 2019-12-07T19:34:10.624+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:10.624+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:10.624+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83eb8300 cost 5444 (1 ios * 4000 + 1444 bytes) 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83eb8300 onodes shared_blobs 0x558e818667e0,0x558e81867960,0x558e8194e1c0 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2828 is 11 SharedBlob(0x558e818667e0 loaded (sbid 0x2828 ref_map(0x3f000~2000=1))) 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x282a is 11 SharedBlob(0x558e81867960 loaded (sbid 0x282a ref_map(0x43000~2000=1))) 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2829 is 11 SharedBlob(0x558e8194e1c0 loaded (sbid 0x2829 ref_map(0x41000~2000=1))) 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83eb8300 allocated 0x[] released 0x[] 2019-12-07T19:34:10.624+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8300 prepare 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83eb8300 2019-12-07T19:34:10.624+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8300 io_done 2019-12-07T19:34:10.624+0000 7fe2c7265700 10 osd.0 33 dequeue_op 0x558e81942b00 finish 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:10.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:10.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1292 < min_flush_size 524288 2019-12-07T19:34:10.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e811dd900 had 1 2019-12-07T19:34:10.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:10.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1315 < min_flush_size 524288 2019-12-07T19:34:10.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1315 < min_flush_size 524288 2019-12-07T19:34:10.624+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x140000 mtime 2019-12-07T19:34:10.621116+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:10.624+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1359aa~523 to file(ino 16 size 0x140000 mtime 2019-12-07T19:34:10.621116+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:10.624+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1359aa 0x1359aa~523 to file(ino 16 size 0x140000 mtime 2019-12-07T19:34:10.621116+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:10.624+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x140000 mtime 2019-12-07T19:34:10.621116+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:10.624+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~110000 x_off 0x1059aa 2019-12-07T19:34:10.624+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x9aa 2019-12-07T19:34:10.624+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:10.624+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xecd and padding block with 0x133 2019-12-07T19:34:10.624+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x135ecd 2019-12-07T19:34:10.624+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:10.624+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:10.624+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:10.624+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:10.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000119358s (0.000000102s flush + 0.000119256s kv commit) 2019-12-07T19:34:10.624+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:10.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:10.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83eb8300> 2019-12-07T19:34:10.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:10.624+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8300 kv_submitted 2019-12-07T19:34:10.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83eb8300 2019-12-07T19:34:10.624+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8300 finishing 2019-12-07T19:34:10.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83eb8300 onodes 2019-12-07T19:34:10.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83eb8300 done 2019-12-07T19:34:10.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:10.624+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83eb8300 [] 2019-12-07T19:34:10.624+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:10.624+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:10.624+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 33/28 ECSubWriteReply(tid=155, last_complete=33'15, committed=1, applied=1)) v2 -- 0x558e83fee000 con 0x558e818d7c00 2019-12-07T19:34:10.624+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.624+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:10.627+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.677+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.727+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.744+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.5 172.21.15.109:0/36936 12 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.744407+0000/34.210730876s send_stamp 34.210730876s delta_ub 0.011681823s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83fee000 con 0x558e8182c800 2019-12-07T19:34:10.744+0000 7fe2e7d3c700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011275656s,0.011681823s]) 2019-12-07T19:34:10.744+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.744407+0000/34.210730876s send_stamp 34.199455220s delta_ub 18446744073.698275960s) v5 -- 0x558e83fee600 con 0x558e8182c800 2019-12-07T19:34:10.744+0000 7fe2e7d3c700 10 osd.0 33 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 32 -> 33 (as per caller) 2019-12-07T19:34:10.744+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.5 172.21.15.109:0/36936 12 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.744407+0000/34.210730876s send_stamp 34.210730876s delta_ub 0.011681823s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83fee000 con 0x558e8181b800 2019-12-07T19:34:10.744+0000 7fe2e7d3c700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011234274s,0.011681823s]) 2019-12-07T19:34:10.744+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.744407+0000/34.210730876s send_stamp 34.199496602s delta_ub 18446744073.698317342s) v5 -- 0x558e84053000 con 0x558e8181b800 2019-12-07T19:34:10.753+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.3 172.21.15.131:0/37076 6 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.753794+0000/34.211306508s send_stamp 34.211306508s delta_ub 0.003015349s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e81984000 2019-12-07T19:34:10.753+0000 7fe2e6d3a700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002638531s,0.003015349s]) 2019-12-07T19:34:10.753+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.753794+0000/34.211306508s send_stamp 34.208667977s delta_ub 18446744073.706913085s) v5 -- 0x558e83e69600 con 0x558e81984000 2019-12-07T19:34:10.753+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.3 172.21.15.131:0/37076 6 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.753794+0000/34.211306508s send_stamp 34.211306508s delta_ub 0.003015349s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a8400 con 0x558e818ff800 2019-12-07T19:34:10.753+0000 7fe2e753b700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002596810s,0.003015349s]) 2019-12-07T19:34:10.753+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37076 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.753794+0000/34.211306508s send_stamp 34.208709698s delta_ub 18446744073.706954806s) v5 -- 0x558e83e68400 con 0x558e818ff800 2019-12-07T19:34:10.777+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.780+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:10.827+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.877+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.927+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:10.954+0000 7fe2c3a5e700 5 osd.0 33 heartbeat osd_stat(store_statfs(0x163fad0000/0x40000000/0x1680000000, data 0x4f9c39/0x52e000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000), peers [1,2,3,4,5,7] op hist []) 2019-12-07T19:34:10.954+0000 7fe2c3a5e700 20 osd.0 33 check_full_status cur ratio 5.6288e-05, physical ratio 5.6288e-05, new state none 2019-12-07T19:34:10.954+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6831/37078,v1:172.21.15.131:6833/37078] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.409739526s delta_ub 18446744073.707539242s) v5 -- 0x558e845c2200 con 0x558e818d7800 2019-12-07T19:34:10.954+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6827/37078,v1:172.21.15.131:6829/37078] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.409739526s delta_ub 18446744073.707539242s) v5 -- 0x558e845c2400 con 0x558e818d7400 2019-12-07T19:34:10.954+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6808/37077,v1:172.21.15.131:6809/37077] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.409739526s delta_ub 18446744073.692672153s) v5 -- 0x558e845c2600 con 0x558e817ff000 2019-12-07T19:34:10.954+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6806/37077,v1:172.21.15.131:6807/37077] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.409739526s delta_ub 18446744073.692672153s) v5 -- 0x558e845c2800 con 0x558e817ff400 2019-12-07T19:34:10.954+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6816/37076,v1:172.21.15.131:6817/37076] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.409739526s delta_ub 18446744073.706954806s) v5 -- 0x558e845c2a00 con 0x558e817ff800 2019-12-07T19:34:10.954+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.131:6814/37076,v1:172.21.15.131:6815/37076] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.409739526s delta_ub 18446744073.706954806s) v5 -- 0x558e845c2c00 con 0x558e817ffc00 2019-12-07T19:34:10.954+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6814/36938,v1:172.21.15.109:6815/36938] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.409739526s delta_ub 0.005831629s) v5 -- 0x558e845c2e00 con 0x558e8181a000 2019-12-07T19:34:10.954+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6812/36938,v1:172.21.15.109:6813/36938] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.409739526s delta_ub 0.005831629s) v5 -- 0x558e845c3000 con 0x558e8181a400 2019-12-07T19:34:10.954+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6806/36936,v1:172.21.15.109:6807/36936] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.409739526s delta_ub 18446744073.698317342s) v5 -- 0x558e845c3200 con 0x558e817fe000 2019-12-07T19:34:10.954+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6804/36936,v1:172.21.15.109:6805/36936] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.409739526s delta_ub 18446744073.698317342s) v5 -- 0x558e845c3400 con 0x558e817fe400 2019-12-07T19:34:10.954+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6830/36939,v1:172.21.15.109:6831/36939] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.409739526s delta_ub 0.019695794s) v5 -- 0x558e845c3600 con 0x558e817fe800 2019-12-07T19:34:10.954+0000 7fe2c3a5e700 1 -- 172.21.15.131:0/37079 --> [v2:172.21.15.109:6828/36939,v1:172.21.15.109:6829/36939] -- osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.409739526s delta_ub 0.019695794s) v5 -- 0x558e845c3800 con 0x558e817fec00 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6806/37077 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.426945650s delta_ub 0.017206124s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e817ff400 2019-12-07T19:34:10.954+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.2 v2:172.21.15.131:6808/37077 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.426899820s delta_ub 0.017160294s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83e69c00 con 0x558e817ff000 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 25 osd.0 33 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:08.655029+0000 last_rx_front 2019-12-07T19:34:08.655013+0000 -> 2019-12-07T19:34:10.955288+0000 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6816/37076 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.412750127s delta_ub 0.003010601s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84053a00 con 0x558e817ff800 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 10 osd.0 33 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 31 -> 33 (as per caller) 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016965816s,0.017206124s]) 2019-12-07T19:34:10.954+0000 7fe2e753b700 25 osd.0 33 handle_osd_ping got reply from osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:08.655029+0000 -> 2019-12-07T19:34:10.955302+0000 last_rx_front 2019-12-07T19:34:10.955288+0000 2019-12-07T19:34:10.954+0000 7fe2e753b700 25 osd.0 33 handle_osd_ping got all replies from osd.2 , erase pending ping(sent at 2019-12-07T19:34:10.955047+0000) and older pending ping(s) 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6827/37078 6 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.412157996s delta_ub 0.002418470s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e818d7400 2019-12-07T19:34:10.954+0000 7fe2e753b700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016906194s,0.017160294s]) 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 25 osd.0 33 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:08.654942+0000 -> 2019-12-07T19:34:10.955335+0000 last_rx_front 2019-12-07T19:34:08.654980+0000 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002723405s,0.003010601s]) 2019-12-07T19:34:10.954+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.3 v2:172.21.15.131:6814/37076 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.412724532s delta_ub 0.002985006s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83e69c00 con 0x558e817ffc00 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 25 osd.0 33 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:08.655063+0000 last_rx_front 2019-12-07T19:34:08.655052+0000 -> 2019-12-07T19:34:10.955347+0000 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002118859s,0.002418470s]) 2019-12-07T19:34:10.954+0000 7fe2e753b700 25 osd.0 33 handle_osd_ping got reply from osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955335+0000 last_rx_front 2019-12-07T19:34:08.654980+0000 -> 2019-12-07T19:34:10.955354+0000 2019-12-07T19:34:10.954+0000 7fe2e753b700 25 osd.0 33 handle_osd_ping got all replies from osd.3 , erase pending ping(sent at 2019-12-07T19:34:10.955047+0000) and older pending ping(s) 2019-12-07T19:34:10.954+0000 7fe2e753b700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.3 up_from 13 peer_clock_delta [0.002678348s,0.002985006s]) 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6830/36939 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.390602411s delta_ub 18446744073.690414501s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e817fe800 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 25 osd.0 33 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:08.655172+0000 -> 2019-12-07T19:34:10.955373+0000 last_rx_front 2019-12-07T19:34:08.655204+0000 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 10 osd.0 33 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 31 -> 33 (as per caller) 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690088879s,18446744073.690414501s]) 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6812/36938 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.404443169s delta_ub 18446744073.704255259s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84053a00 con 0x558e8181a400 2019-12-07T19:34:10.954+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.1 v2:172.21.15.131:6831/37078 6 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.412104855s delta_ub 0.002365329s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83e69c00 con 0x558e818d7800 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 25 osd.0 33 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:08.655222+0000 last_rx_front 2019-12-07T19:34:08.655237+0000 -> 2019-12-07T19:34:10.955392+0000 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 1 -- 172.21.15.131:0/37079 <== osd.4 v2:172.21.15.109:6814/36938 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.404396685s delta_ub 18446744073.704208775s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e8181a000 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 10 osd.0 33 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 32 -> 33 (as per caller) 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703910302s,18446744073.704255259s]) 2019-12-07T19:34:10.954+0000 7fe2e753b700 25 osd.0 33 handle_osd_ping got reply from osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:08.655063+0000 -> 2019-12-07T19:34:10.955408+0000 last_rx_front 2019-12-07T19:34:10.955347+0000 2019-12-07T19:34:10.954+0000 7fe2e753b700 25 osd.0 33 handle_osd_ping got all replies from osd.1 , erase pending ping(sent at 2019-12-07T19:34:10.955047+0000) and older pending ping(s) 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.7 v2:172.21.15.109:6828/36939 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.390681783s delta_ub 18446744073.690493873s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84053a00 con 0x558e817fec00 2019-12-07T19:34:10.954+0000 7fe2e753b700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002005040s,0.002365329s]) 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 25 osd.0 33 handle_osd_ping got reply from osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:08.655222+0000 -> 2019-12-07T19:34:10.955428+0000 last_rx_front 2019-12-07T19:34:10.955392+0000 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 25 osd.0 33 handle_osd_ping got all replies from osd.4 , erase pending ping(sent at 2019-12-07T19:34:10.955047+0000) and older pending ping(s) 2019-12-07T19:34:10.954+0000 7fe2e6d3a700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703827799s,18446744073.704208775s]) 2019-12-07T19:34:10.954+0000 7fe2e753b700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6804/36936 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.421500869s delta_ub 0.011761343s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83e69c00 con 0x558e817fe400 2019-12-07T19:34:10.954+0000 7fe2e753b700 25 osd.0 33 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:08.655297+0000 last_rx_front 2019-12-07T19:34:08.655192+0000 -> 2019-12-07T19:34:10.955444+0000 2019-12-07T19:34:10.954+0000 7fe2e753b700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011365216s,0.011761343s]) 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 25 osd.0 33 handle_osd_ping got reply from osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955373+0000 last_rx_front 2019-12-07T19:34:08.655204+0000 -> 2019-12-07T19:34:10.955455+0000 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 25 osd.0 33 handle_osd_ping got all replies from osd.7 , erase pending ping(sent at 2019-12-07T19:34:10.955047+0000) and older pending ping(s) 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690086588s,18446744073.690493873s]) 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 1 -- 172.21.15.131:0/37079 <== osd.5 v2:172.21.15.109:6806/36936 10 ==== osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:10.955047+0000/34.409739526s send_stamp 34.421560254s delta_ub 0.011820728s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84053a00 con 0x558e817fe000 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 25 osd.0 33 handle_osd_ping got reply from osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:08.655297+0000 -> 2019-12-07T19:34:10.955481+0000 last_rx_front 2019-12-07T19:34:10.955444+0000 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 25 osd.0 33 handle_osd_ping got all replies from osd.5 , erase pending ping(sent at 2019-12-07T19:34:10.955047+0000) and older pending ping(s) 2019-12-07T19:34:10.954+0000 7fe2e7d3c700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011387419s,0.011820728s]) 2019-12-07T19:34:10.977+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.027+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.077+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.128+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.178+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.228+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.278+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.328+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.378+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.428+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.461+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.4 172.21.15.109:0/36938 5 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.461505+0000/34.910731969s send_stamp 34.910731969s delta_ub 18446744073.704255259s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e81985400 2019-12-07T19:34:11.461+0000 7fe2e6d3a700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703795472s,18446744073.704255259s]) 2019-12-07T19:34:11.461+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.461505+0000/34.910731969s send_stamp 34.916488113s delta_ub 0.005756144s) v5 -- 0x558e845c3600 con 0x558e81985400 2019-12-07T19:34:11.461+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.4 172.21.15.109:0/36938 5 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.461505+0000/34.910731969s send_stamp 34.910731969s delta_ub 18446744073.704255259s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84053a00 con 0x558e81985800 2019-12-07T19:34:11.461+0000 7fe2e7d3c700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703774806s,18446744073.704255259s]) 2019-12-07T19:34:11.461+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.461505+0000/34.910731969s send_stamp 34.916508779s delta_ub 0.005776810s) v5 -- 0x558e84053400 con 0x558e81985800 2019-12-07T19:34:11.478+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.502+0000 7fe2e2ca2700 10 osd.0 33 tick_without_osd_lock 2019-12-07T19:34:11.502+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fa80000/0x40000000/0x1680000000, data 0x5482be/0x57e000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:11.502+0000 7fe2e2ca2700 25 osd.0 33 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955408+0000 last_rx_front 2019-12-07T19:34:10.955347+0000 2019-12-07T19:34:11.502+0000 7fe2e2ca2700 25 osd.0 33 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955302+0000 last_rx_front 2019-12-07T19:34:10.955288+0000 2019-12-07T19:34:11.502+0000 7fe2e2ca2700 25 osd.0 33 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955335+0000 last_rx_front 2019-12-07T19:34:10.955354+0000 2019-12-07T19:34:11.502+0000 7fe2e2ca2700 25 osd.0 33 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955428+0000 last_rx_front 2019-12-07T19:34:10.955392+0000 2019-12-07T19:34:11.502+0000 7fe2e2ca2700 25 osd.0 33 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955481+0000 last_rx_front 2019-12-07T19:34:10.955444+0000 2019-12-07T19:34:11.502+0000 7fe2e2ca2700 25 osd.0 33 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955373+0000 last_rx_front 2019-12-07T19:34:10.955455+0000 2019-12-07T19:34:11.502+0000 7fe2e2ca2700 20 osd.0 33 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:11.502+0000 7fe2e2ca2700 10 osd.0 33 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:11.502+0000 7fe2e2ca2700 20 osd.0 33 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:11.502+0000 7fe2e2ca2700 10 osd.0 33 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:11.529+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.563+0000 7fe2e4535700 10 osd.0 33 tick 2019-12-07T19:34:11.563+0000 7fe2e4535700 10 osd.0 33 do_waiters -- start 2019-12-07T19:34:11.563+0000 7fe2e4535700 10 osd.0 33 do_waiters -- finish 2019-12-07T19:34:11.563+0000 7fe2e4535700 20 osd.0 33 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:11.573+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.7 172.21.15.109:0/36939 10 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.009373375s delta_ub 18446744073.690493873s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee800 con 0x558e81748c00 2019-12-07T19:34:11.573+0000 7fe2e6d3a700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.690037380s,18446744073.690493873s]) 2019-12-07T19:34:11.573+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.028887611s delta_ub 0.019514236s) v5 -- 0x558e845c3600 con 0x558e81748c00 2019-12-07T19:34:11.573+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.7 172.21.15.109:0/36939 10 ==== osd_ping(ping e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.009373375s delta_ub 18446744073.690493873s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee800 con 0x558e81749000 2019-12-07T19:34:11.573+0000 7fe2e6d3a700 20 osd.0 33 handle_osd_ping new stamps hbstamp(osd.7 up_from 13 peer_clock_delta [18446744073.689995278s,18446744073.690493873s]) 2019-12-07T19:34:11.573+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36939 -- osd_ping(ping_reply e33 up_from 13 ping_stamp 2019-12-07T19:34:11.573934+0000/35.009373375s send_stamp 35.028929713s delta_ub 0.019556338s) v5 -- 0x558e845c2e00 con 0x558e81749000 2019-12-07T19:34:11.578+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 607422 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.609+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 249 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558e83ebbc00 con 0x558e818d7c00 2019-12-07T19:34:11.609+0000 7fe2e7d3c700 15 osd.0 33 enqueue_peering_evt 1.8s5 epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:11.609+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.609+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.609+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.609+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.609+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e34) pg 0x558e841a4000 2019-12-07T19:34:11.609+0000 7fe2c7a66700 20 osd.0 op_wq _add_slot_waiter 1.8s5 peering, item epoch is 34, will wait on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.609+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:11.610+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 132 ==== pg_info2(1.fs1 1.fs1 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558e83ebbc00 con 0x558e818d7000 2019-12-07T19:34:11.610+0000 7fe2e7d3c700 15 osd.0 33 enqueue_peering_evt 1.fs1 epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:11.610+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.610+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.610+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.610+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.610+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e34) pg 0x558e81a9b000 2019-12-07T19:34:11.610+0000 7fe2c8267700 20 osd.0 op_wq _add_slot_waiter 1.fs1 peering, item epoch is 34, will wait on OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.610+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:11.611+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 250 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558e83ebbc00 con 0x558e818d7c00 2019-12-07T19:34:11.611+0000 7fe2e7d3c700 15 osd.0 33 enqueue_peering_evt 1.es4 epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:11.611+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.611+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.611+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.611+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.611+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e34) pg 0x558e81a98000 2019-12-07T19:34:11.611+0000 7fe2c8a68700 20 osd.0 op_wq _add_slot_waiter 1.es4 peering, item epoch is 34, will wait on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.611+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:11.611+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 133 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558e83ebbc00 con 0x558e818d7000 2019-12-07T19:34:11.611+0000 7fe2e7d3c700 15 osd.0 33 enqueue_peering_evt 1.1s1 epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:11.611+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.611+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 251 ==== pg_info2(1.as3 1.as3 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558e83ebbc00 con 0x558e818d7c00 2019-12-07T19:34:11.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e34) pg 0x558e819e4000 2019-12-07T19:34:11.611+0000 7fe2cb26d700 20 osd.0 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 34, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.611+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.611+0000 7fe2e7d3c700 15 osd.0 33 enqueue_peering_evt 1.as3 epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:11.611+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.611+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.611+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.611+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.611+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e34) pg 0x558e8193b000 2019-12-07T19:34:11.611+0000 7fe2c6a64700 20 osd.0 op_wq _add_slot_waiter 1.as3 peering, item epoch is 34, will wait on OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.611+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:11.611+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 252 ==== pg_info2(1.7s3 1.7s3 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558e83ebbc00 con 0x558e818d7c00 2019-12-07T19:34:11.611+0000 7fe2e7d3c700 15 osd.0 33 enqueue_peering_evt 1.7s3 epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:11.611+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.612+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.612+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.612+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.612+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e34) pg 0x558e8122e000 2019-12-07T19:34:11.612+0000 7fe2c425f700 20 osd.0 op_wq _add_slot_waiter 1.7s3 peering, item epoch is 34, will wait on OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.612+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:11.613+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 253 ==== osd_map(34..34 src has 1..34) v4 ==== 661+0+0 (unknown 0 0 0) 0x558e8178eb40 con 0x558e818d7c00 2019-12-07T19:34:11.613+0000 7fe2d4a80700 20 osd.0 33 OSD::ms_dispatch: osd_map(34..34 src has 1..34) v4 2019-12-07T19:34:11.613+0000 7fe2d4a80700 10 osd.0 33 do_waiters -- start 2019-12-07T19:34:11.613+0000 7fe2d4a80700 10 osd.0 33 do_waiters -- finish 2019-12-07T19:34:11.613+0000 7fe2d4a80700 20 osd.0 33 _dispatch 0x558e8178eb40 osd_map(34..34 src has 1..34) v4 2019-12-07T19:34:11.613+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 254 ==== MOSDECSubOpWrite(1.9s3 34/28 ECSubWrite(tid=156, reqid=osd.1.0:156, at_version=34'17, trim_to=0'0, roll_forward_to=33'15)) v2 ==== 2632+0+0 (unknown 0 0 0) 0x558e83fec800 con 0x558e818d7c00 2019-12-07T19:34:11.613+0000 7fe2e7d3c700 15 osd.0 33 enqueue_op 0x558e81942420 prio 127 cost 0 latency 0.000028 epoch 34 MOSDECSubOpWrite(1.9s3 34/28 ECSubWrite(tid=156, reqid=osd.1.0:156, at_version=34'17, trim_to=0'0, roll_forward_to=33'15)) v2 2019-12-07T19:34:11.613+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 34/28 ECSubWrite(tid=156, reqid=osd.1.0:156, at_version=34'17, trim_to=0'0, roll_forward_to=33'15)) v2) prio 127 cost 0 e34) 2019-12-07T19:34:11.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 34/28 ECSubWrite(tid=156, reqid=osd.1.0:156, at_version=34'17, trim_to=0'0, roll_forward_to=33'15)) v2) prio 127 cost 0 e34) queued 2019-12-07T19:34:11.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 34/28 ECSubWrite(tid=156, reqid=osd.1.0:156, at_version=34'17, trim_to=0'0, roll_forward_to=33'15)) v2) prio 127 cost 0 e34) pg 0x558e8192b000 2019-12-07T19:34:11.613+0000 7fe2c7265700 10 osd.0 33 dequeue_op 0x558e81942420 prio 127 cost 0 latency 0.000058 MOSDECSubOpWrite(1.9s3 34/28 ECSubWrite(tid=156, reqid=osd.1.0:156, at_version=34'17, trim_to=0'0, roll_forward_to=33'15)) v2 pg pg[1.9s3( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] 2019-12-07T19:34:11.613+0000 7fe2c7265700 10 osd.0 33 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 33 -> 34 (as per caller) 2019-12-07T19:34:11.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 33'15 (0'0,33'15] local-lis/les=28/29 n=4 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] _handle_message: MOSDECSubOpWrite(1.9s3 34/28 ECSubWrite(tid=156, reqid=osd.1.0:156, at_version=34'17, trim_to=0'0, roll_forward_to=33'15)) v2 2019-12-07T19:34:11.613+0000 7fe2d4a80700 3 osd.0 33 handle_osd_map epochs [34,34], i have 33, src has [1,34] 2019-12-07T19:34:11.613+0000 7fe2c7265700 20 snap_mapper.remove_oid 1:9b8adabe:::smithi13140796-6:a 2019-12-07T19:34:11.613+0000 7fe2c7265700 20 snap_mapper._remove_oid 1:9b8adabe:::smithi13140796-6:a 2019-12-07T19:34:11.613+0000 7fe2d4a80700 10 osd.0 33 handle_osd_map got inc map for epoch 34 2019-12-07T19:34:11.613+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:11.613+0000 7fe2c7265700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:11.613+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3478e826:::osdmap.33:0# 0x0~0 2019-12-07T19:34:11.613+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1438 size 0x1438 (5176) 2019-12-07T19:34:11.613+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:11.613+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84755420 blob([0x2b9000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e847553b0 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:11.613+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:11.613+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84755340 blob([!~1000,0x2ba000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,438]) SharedBlob(0x558e847552d0 sbid 0x0)) need 0x1000~438 cache has 0x[1000~438] 2019-12-07T19:34:11.613+0000 7fe2c7265700 20 snap_mapper.get_snaps 1:9b8adabe:::smithi13140796-6:a a 2019-12-07T19:34:11.613+0000 7fe2c7265700 20 snap_mapper.clear_snaps 1:9b8adabe:::smithi13140796-6:a 2019-12-07T19:34:11.613+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3478e826:::osdmap.33:0# 0x0~1438 = 5176 2019-12-07T19:34:11.613+0000 7fe2d4a80700 10 osd.0 33 add_map_bl 33 5176 bytes 2019-12-07T19:34:11.613+0000 7fe2c7265700 20 snap_mapper.clear_snaps rm OBJ_.3_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:11.613+0000 7fe2c7265700 20 snap_mapper._remove_oid rm SNA_1_000000000000000A_.3_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:11.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 33'15 (0'0,33'15] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] append_log log((0'0,33'15], crt=33'15) [34'16 (33'14) delete 1:9b8adabe:::smithi13140796-6:a by unknown.0.0:0 2019-12-07T19:34:11.613853+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,34'17 (33'15) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:11.613853+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:11.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 34'16 (0'0,34'16] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=33'15 crt=33'15 lcod 33'15 active mbc={} ps=[4~1,6~2]] add_log_entry 34'16 (33'14) delete 1:9b8adabe:::smithi13140796-6:a by unknown.0.0:0 2019-12-07T19:34:11.613853+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:11.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=33'15 crt=33'15 lcod 33'15 active mbc={} ps=[4~1,6~2]] add_log_entry 34'17 (33'15) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:11.613853+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:11.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=33'15 crt=33'15 lcod 33'15 active mbc={} ps=[4~1,6~2]] append_log approx pg log length = 17 2019-12-07T19:34:11.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=33'15 crt=33'15 lcod 33'15 active mbc={} ps=[4~1,6~2]] append_log transaction_applied = 1 2019-12-07T19:34:11.613+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:11.613+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 34'16, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:11.614+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83eb8600 seq 37 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) get_onode oid 3#1:9b8adabe:::smithi13140796-6:a#10 key 0x8380000000000000019b8adabe21736d'ithi13140796-6!='0x000000000000000a0000000000000010'o' 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 osd.0 33 got_full_map 34, nothing requested 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.9s3_head 0x558e819b0b40) r -2 v.len 0 2019-12-07T19:34:11.614+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:a# -> 3#1:9b8adabe:::smithi13140796-6:a#10 2019-12-07T19:34:11.614+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:a# -> 3#1:9b8adabe:::smithi13140796-6:a#10 = 0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 osd.0 33 handle_osd_map pg_num_history pg_num_history(e34 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps purged_snaps {34={}} 2019-12-07T19:34:11.614+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 2 keys 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 osd.0 33 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e34 [1,34] lci=[13,34]) 2019-12-07T19:34:11.614+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# 2 keys = 0 2019-12-07T19:34:11.614+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) 2019-12-07T19:34:11.614+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:11.614+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.OBJ_.3_0000000000000001.9D15B5D7.a.smithi13140796-6..' <- OBJ_.3_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e8441a000 seq 26 2019-12-07T19:34:11.614+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c76b972f:::inc_osdmap.34:0# key 0x7f7fffffffffffffffc76b97'/!inc_osdmap.34!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c76b972f:::inc_osdmap.34:0# 0x0~269 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1160 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c76b972f:::inc_osdmap.34:0# 0x0~269 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~269 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xd97 on front/back, now 0x0~1000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8441a000 1 blobs 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x308000~1000/1000,1000,0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x308000~1000] 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8423caf0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3110 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8423caf0 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3110 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.blob(0x558e8423caf0) get_ref 0x0~269 Blob(0x558e8423caf0 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd3110 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.blob(0x558e8423caf0) get_ref init 0x1000, 1000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~269: 0x0~269 Blob(0x558e8423caf0 blob([0x308000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x269) SharedBlob(0x558e83dd3110 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e83dd3128 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x269 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8402fc88) dirty_range mark inline shard dirty 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c76b972f:::inc_osdmap.34:0# 0x0~269 = 0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3ff8e826:::osdmap.34:0# key 0x7f7fffffffffffffff3ff8e8'&!osdmap.34!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3ff8e826:::osdmap.34:0# 0x0~1464 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1161 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3ff8e826:::osdmap.34:0# 0x0~1464 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~464 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xb9c on front/back, now 0x0~1000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8441a000 2 blobs 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x4ba000~2000/1000,2000,0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x4ba000~2000] 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6c690 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84755180 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6c690 blob([0x4ba000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755180 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.blob(0x558e83e6c690) get_ref 0x0~1000 Blob(0x558e83e6c690 blob([0x4ba000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84755180 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.blob(0x558e83e6c690) get_ref init 0x1000, 1000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e83e6c690 blob([0x4ba000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e84755180 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e84755198 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84755110 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e847550a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84755110 blob([!~1000,0x4bb000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e847550a0 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.blob(0x558e84755110) get_ref 0x1000~464 Blob(0x558e84755110 blob([!~1000,0x4bb000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e847550a0 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.blob(0x558e84755110) get_ref init 0x2000, 1000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~464: 0x1000~464 Blob(0x558e84755110 blob([!~1000,0x4bb000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558e847550a0 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e847550b8 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1464 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84314608) dirty_range mark inline shard dirty 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3ff8e826:::osdmap.34:0# 0x0~1464 = 0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8441a000 0 blobs 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8441a000 0 blobs 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8441a000 cost 23318 (4 ios * 4000 + 7318 bytes) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8441a000 onodes 0x558e80251b80,0x558e817b9180,0x558e8402fb80,0x558e84314500 shared_blobs 2019-12-07T19:34:11.614+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys rm 0x00000000000000000000000000000402'.SNA_1_000000000000000A_.3_0000000000000001.9D15B5D7.a.smithi13140796-6..' <- SNA_1_000000000000000A_.3_0000000000000001.9D15B5D7.a.smithi13140796-6.. 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:11.614+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:11.614+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8402fc88) update #-1:c76b972f:::inc_osdmap.34:0# 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000034.00000000000000000016' <- 0000000034.00000000000000000016 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e8402fc88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c76b972f:::inc_osdmap.34:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.0000000034.00000000000000000017' <- 0000000034.00000000000000000017 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84314608) update #-1:3ff8e826:::osdmap.34:0# 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84314608) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:11.614+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3ff8e826:::osdmap.34:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:11.614+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83eb8600 cost 7506 (1 ios * 4000 + 3506 bytes) 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83eb8600 onodes 0x558e83c72c80,0x558e8402e500 shared_blobs 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore.extentmap(0x558e83c72d88) update 3#1:9b8adabe:::smithi13140796-6:head# 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8441a000 allocated 0x[308000~1000,4ba000~2000] released 0x[] 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 freelist allocate 0x308000~1000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 freelist _xor first_key 0x300000 last_key 0x300000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 freelist allocate 0x4ba000~2000 2019-12-07T19:34:11.614+0000 7fe2d4a80700 20 freelist _xor first_key 0x480000 last_key 0x480000 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9b8adabe:::smithi13140796-6:head# is 55136 (54389 bytes onode + 2 bytes spanning blobs + 745 bytes inline extents) 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441a000 prepare 2019-12-07T19:34:11.614+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8441a000 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore.extentmap(0x558e8402e608) update 3#1:9b8adabe:::smithi13140796-6:a#10 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 3#1:9b8adabe:::smithi13140796-6:a#10 is 500 (410 bytes onode + 2 bytes spanning blobs + 88 bytes inline extents) 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83eb8600 allocated 0x[] released 0x[] 2019-12-07T19:34:11.614+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8600 prepare 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83eb8600 2019-12-07T19:34:11.614+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8600 io_done 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.614+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 lua=33'15 crt=33'15 lcod 33'15 active mbc={} ps=[4~1,6~2]] op_applied version 34'17 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8499 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7fe2c7265700 10 osd.0 33 dequeue_op 0x558e81942420 finish 2019-12-07T19:34:11.614+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441a000 aio_wait 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.614+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8441a000 2019-12-07T19:34:11.614+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441a000 io_done 2019-12-07T19:34:11.614+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 41267 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 58080 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72c80 had 1 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402e500 had 1 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84770000 had 1 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3859546112 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998457) bluestore 86 GiB free (0.95994), bluefs_ratio 0.0399435 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 58103 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 58103 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x140000 mtime 2019-12-07T19:34:10.621116+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x135ecd~e2f7 to file(ino 16 size 0x140000 mtime 2019-12-07T19:34:10.621116+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x135ecd 0x135ecd~e2f7 to file(ino 16 size 0x140000 mtime 2019-12-07T19:34:10.621116+0000 allocated 140000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~110000]) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _allocate len 0x41c4 from 0 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _allocate len 0x41c4 from 1 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd270000 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd270000~10000/10000,10000,acd270000 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x150000 to include allocated 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 39 (was clean) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x105ecd 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xecd 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x1c4 and padding block with 0xe3c 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x150000 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1441c4 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (39) on file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]), flushing log 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 39 len 0x49 crc 0x2cac745a) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x26000~1000 to file(ino 1 size 0x26000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x26000 0x26000~1000 to file(ino 1 size 0x26000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x27000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x26000 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x27000 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000020 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 39 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000462310s (0.000000421s flush + 0.000461889s kv commit) 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83eb8600> 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.614+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8600 kv_submitted 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83eb8600 2019-12-07T19:34:11.614+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8600 finishing 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83eb8600 onodes 0x558e83c72c80,0x558e8402e500 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83eb8600 done 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8818 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.614+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83eb8600 [] 2019-12-07T19:34:11.614+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402fb80 had 1 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84314500 had 1 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8841 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8841 < min_flush_size 524288 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.614+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 33/28 ECSubWriteReply(tid=156, last_complete=34'17, committed=1, applied=1)) v2 -- 0x558e83fee400 con 0x558e818d7c00 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1441c4~2289 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1441c4 0x1441c4~2289 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x1141c4 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x1c4 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x44d and padding block with 0xbb3 2019-12-07T19:34:11.614+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x14644d 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:11.614+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000105327s (0.000003694s flush + 0.000101633s kv commit) 2019-12-07T19:34:11.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8441a000> 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.614+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441a000 kv_submitted 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8441a000 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e8441a000 osr 0x558e8107a500 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e845eedc0) _discard drop 23 0x1b000~1000 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e845eedc0) prepare_write seq 24 0x1b000~1000 crc b83c5ff7 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e845eedc0) _discard drop 23 0x2000~1000 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e845eedc0) prepare_write seq 24 0x2000~1000 crc 152f167f 2019-12-07T19:34:11.614+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:11.615+0000 7fe2d7285700 10 osd.0 33 _committed_osd_maps 34..34 2019-12-07T19:34:11.615+0000 7fe2d7285700 10 osd.0 33 advance to epoch 34 (<= last 34 <= newest_map 34) 2019-12-07T19:34:11.615+0000 7fe2d7285700 7 osd.0 34 consume_map version 34 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 34 identify_splits_and_merges 1.8s5 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 34 identify_splits_and_merges 1.9s3 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 34 identify_splits_and_merges 1.1s1 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 34 identify_splits_and_merges 1.as3 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 34 identify_splits_and_merges 1.2s0 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 34 identify_splits_and_merges 1.3s5 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 34 identify_splits_and_merges 1.bs1 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 34 identify_splits_and_merges 1.4s1 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 34 identify_splits_and_merges 1.5s4 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 34 identify_splits_and_merges 1.ds5 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 34 identify_splits_and_merges 1.es4 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 34 identify_splits_and_merges 1.7s3 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 34 identify_splits_and_merges 1.fs1 e33 to e34 pg_nums {18=16} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 34 prune_pg_created 2019-12-07T19:34:11.615+0000 7fe2d7285700 10 osd.0:0.consume_map 34 (was 33) 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 pending_peering first epoch 34 <= 34, requeueing 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {34=} 2019-12-07T19:34:11.615+0000 7fe2d7285700 10 osd.0:1.consume_map 34 (was 33) 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 pending_peering first epoch 34 <= 34, requeueing 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {34=} 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2d7285700 10 osd.0:2.consume_map 34 (was 33) 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 pending_peering first epoch 34 <= 34, requeueing 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.as3 to_process <> waiting <> waiting_peering {34=} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e34) pg 0x558e841a4000 2019-12-07T19:34:11.615+0000 7fe2d7285700 10 osd.0:3.consume_map 34 (was 33) 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 2019-12-07T19:34:11.615+0000 7fe2d7285700 10 osd.0:4.consume_map 34 (was 33) 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:11.615+0000 7fe2d7285700 10 osd.0:5.consume_map 34 (was 33) 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:11.615+0000 7fe2d7285700 10 osd.0:6.consume_map 34 (was 33) 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 pending_peering first epoch 34 <= 34, requeueing 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {34=} 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2d7285700 10 osd.0:7.consume_map 34 (was 33) 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e34) pg 0x558e819e4000 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 pending_peering first epoch 34 <= 34, requeueing 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.7s3 to_process <> waiting <> waiting_peering {34=} 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 pending_peering first epoch 34 <= 34, requeueing 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 osd.0 pg_epoch: 33 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.fs1 to_process <> waiting <> waiting_peering {34=} 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 osd.0:0.update_pg_epoch 1.8s5 33 -> 34 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e34) pg 0x558e81a98000 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 osd.0 pg_epoch: 33 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,4,5,2,3,0]/[1,4,5,2,3,0] -- 1/1 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.1s1 33 -> 34 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 osd.0 pg_epoch: 33 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] handle_advance_map [3,0,2,4,5,1]/[3,0,2,4,5,1] -- 3/3 2019-12-07T19:34:11.615+0000 7fe2d7285700 10 osd.0 34 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 osd.0 pg_epoch: 34 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7fe2d7285700 15 osd.0 34 enqueue_peering_evt 1.8s5 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 osd.0 pg_epoch: 34 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 osd.0 pg_epoch: 34 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.8s5_head options 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 osd.0 pg_epoch: 33 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 osd.0 pg_epoch: 34 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 osd.0:6.update_pg_epoch 1.es4 33 -> 34 2019-12-07T19:34:11.615+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:34:11.615+0000 7fe2d7285700 15 osd.0 34 enqueue_peering_evt 1.9s3 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 osd.0 pg_epoch: 34 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 osd.0 pg_epoch: 33 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,3,4,5,0,2]/[1,3,4,5,0,2] -- 1/1 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 osd.0 pg_epoch: 34 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7fe2d7285700 15 osd.0 34 enqueue_peering_evt 1.1s1 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 osd.0 pg_epoch: 34 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 osd.0 pg_epoch: 34 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 pg_epoch: 34 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 osd.0 pg_epoch: 34 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 osd.0 pg_epoch: 34 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 pg_epoch: 34 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 osd.0 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558e8192b000 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 osd.0 pg_epoch: 34 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 osd.0 pg_epoch: 34 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 osd.0 pg_epoch: 34 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 osd.0 pg_epoch: 34 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 osd.0 pg_epoch: 34 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 osd.0 pg_epoch: 34 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7fe2d7285700 15 osd.0 34 enqueue_peering_evt 1.as3 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 osd.0 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 osd.0 pg_epoch: 34 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 osd.0 pg_epoch: 34 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7fe2d7285700 15 osd.0 34 enqueue_peering_evt 1.2s0 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 osd.0 pg_epoch: 34 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 pg_epoch: 34 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2d7285700 15 osd.0 34 enqueue_peering_evt 1.3s5 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 osd.0 pg_epoch: 34 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 osd.0 pg_epoch: 34 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2cb26d700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 osd.0 pg_epoch: 34 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.9s3 33 -> 34 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 osd.0 pg_epoch: 34 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e34) pg 0x558e81a9b000 2019-12-07T19:34:11.615+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 osd.0 pg_epoch: 34 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 osd.0 pg_epoch: 34 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2cb26d700 10 osd.0 pg_epoch: 33 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:11.615+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558e81231000 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 osd.0 pg_epoch: 34 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:11.615+0000 7fe2cb26d700 10 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 33 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 osd.0 pg_epoch: 34 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e83eb8900 seq 32 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.fs1 33 -> 34 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e8441af00 seq 40 2019-12-07T19:34:11.615+0000 7fe2cb26d700 10 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:11.615+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 33 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] handle_advance_map [3,0,5,4,1,2]/[3,0,5,4,1,2] -- 3/3 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._biginfo' <- _biginfo 2019-12-07T19:34:11.615+0000 7fe2ca26b700 10 osd.0 pg_epoch: 33 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:11.615+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.3s5 33 -> 34 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._epoch' <- _epoch 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:11.615+0000 7fe2cb26d700 10 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e83e72c00 seq 28 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:11.615+0000 7fe2ca26b700 10 osd.0 pg_epoch: 33 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83eb8900 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8441af00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8441af00 onodes shared_blobs 2019-12-07T19:34:11.615+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83eb8900 onodes shared_blobs 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8441af00 allocated 0x[] released 0x[] 2019-12-07T19:34:11.615+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83eb8900 allocated 0x[] released 0x[] 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7fe2ca26b700 10 osd.0 pg_epoch: 34 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441af00 prepare 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8900 prepare 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8441af00 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:11.615+0000 7fe2cba6e700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441af00 io_done 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 pg_epoch: 34 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83eb8900 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8900 io_done 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:11.615+0000 7fe2ca26b700 10 osd.0 pg_epoch: 34 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72c00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72c00 onodes shared_blobs 2019-12-07T19:34:11.615+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.3s5_head options 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 pg_epoch: 34 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72c00 allocated 0x[] released 0x[] 2019-12-07T19:34:11.615+0000 7fe2cb26d700 10 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2ca26b700 10 osd.0 pg_epoch: 34 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7fe2cb26d700 10 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72c00 prepare 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72c00 2019-12-07T19:34:11.615+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72c00 io_done 2019-12-07T19:34:11.615+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558e841a4000 2019-12-07T19:34:11.615+0000 7fe2ca26b700 20 osd.0 pg_epoch: 34 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 33 while current is 34 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2ca26b700 20 osd.0 pg_epoch: 34 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 pg_epoch: 34 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7fe2ca26b700 20 osd.0 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2cb26d700 10 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c7a66700 10 osd.0 pg_epoch: 34 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:11.615+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2d7285700 15 osd.0 34 enqueue_peering_evt 1.bs1 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2d7285700 15 osd.0 34 enqueue_peering_evt 1.4s1 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2d7285700 15 osd.0 34 enqueue_peering_evt 1.5s4 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:11.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e819a0900 seq 38 2019-12-07T19:34:11.615+0000 7fe2d7285700 15 osd.0 34 enqueue_peering_evt 1.ds5 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:11.615+0000 7fe2ca26b700 10 osd.0 pg_epoch: 34 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:11.615+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558e83c78000 2019-12-07T19:34:11.615+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:34:11.615+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2ca26b700 10 osd.0 pg_epoch: 34 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558e81938000 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:11.615+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558e83d07000 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:11.615+0000 7fe2ca26b700 20 osd.0 pg_epoch: 34 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0900 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.615+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0900 onodes shared_blobs 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0900 allocated 0x[] released 0x[] 2019-12-07T19:34:11.615+0000 7fe2ca26b700 10 osd.0 pg_epoch: 34 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 prepare 2019-12-07T19:34:11.615+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0900 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 io_done 2019-12-07T19:34:11.615+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2c9269700 10 osd.0 pg_epoch: 33 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7fe2c5a62700 10 osd.0 pg_epoch: 33 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 33 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.5s4 33 -> 34 2019-12-07T19:34:11.615+0000 7fe2c5a62700 20 osd.0:4.update_pg_epoch 1.4s1 33 -> 34 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c6263700 20 osd.0:3.update_pg_epoch 1.bs1 33 -> 34 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e34) pg 0x558e8122e000 2019-12-07T19:34:11.615+0000 7fe2c9269700 10 osd.0 pg_epoch: 33 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] handle_advance_map [2,1,3,4,0,5]/[2,1,3,4,0,5] -- 2/2 2019-12-07T19:34:11.615+0000 7fe2c5a62700 10 osd.0 pg_epoch: 33 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:11.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 33 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:11.615+0000 7fe2c9269700 10 osd.0 pg_epoch: 34 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 34 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7fe2c5a62700 10 osd.0 pg_epoch: 34 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 33 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.7s3 33 -> 34 2019-12-07T19:34:11.615+0000 7fe2c9269700 10 osd.0 pg_epoch: 34 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 34 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7fe2c5a62700 10 osd.0 pg_epoch: 34 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.5s4_head options 2019-12-07T19:34:11.615+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:34:11.615+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs1_head options 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 33 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:11.615+0000 7fe2c9269700 10 osd.0 pg_epoch: 34 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7fe2c5a62700 10 osd.0 pg_epoch: 34 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 34 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7fe2c9269700 20 osd.0 pg_epoch: 34 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7fe2c5a62700 20 osd.0 pg_epoch: 34 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7fe2c6263700 20 osd.0 pg_epoch: 34 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7fe2c9269700 20 osd.0 pg_epoch: 34 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7fe2c9269700 20 osd.0 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7fe2c5a62700 20 osd.0 pg_epoch: 34 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7fe2c6263700 20 osd.0 pg_epoch: 34 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:34:11.615+0000 7fe2c5a62700 20 osd.0 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7fe2c6263700 20 osd.0 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7fe2c9269700 10 osd.0 pg_epoch: 34 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7fe2c5a62700 10 osd.0 pg_epoch: 34 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 34 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7fe2c9269700 10 osd.0 pg_epoch: 34 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 pg_epoch: 34 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7fe2c5a62700 10 osd.0 pg_epoch: 34 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 34 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7fe2c9269700 20 osd.0 pg_epoch: 34 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 pg_epoch: 34 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7fe2c5a62700 20 osd.0 pg_epoch: 34 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7fe2c6263700 20 osd.0 pg_epoch: 34 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c9269700 10 osd.0 pg_epoch: 34 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2c5a62700 10 osd.0 pg_epoch: 34 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 34 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2d7285700 15 osd.0 34 enqueue_peering_evt 1.es4 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 pg_epoch: 34 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2d7285700 15 osd.0 34 enqueue_peering_evt 1.7s3 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558e819e4000 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.615+0000 7fe2cb26d700 10 osd.0 pg_epoch: 34 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2d7285700 15 osd.0 34 enqueue_peering_evt 1.fs1 epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:11.615+0000 7fe2d7285700 7 osd.0 34 activate_map version 34 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e819a0c00 seq 38 2019-12-07T19:34:11.615+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:34:11.615+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:11.615+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:11.615+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0c00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.615+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558e819e7000 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0c00 onodes shared_blobs 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0c00 allocated 0x[] released 0x[] 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 prepare 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0c00 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 io_done 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c5261700 10 osd.0 pg_epoch: 33 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558e8122e000 2019-12-07T19:34:11.615+0000 7fe2c5261700 20 osd.0:5.update_pg_epoch 1.ds5 33 -> 34 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c5261700 10 osd.0 pg_epoch: 33 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558e81a9b000 2019-12-07T19:34:11.615+0000 7fe2c5261700 10 osd.0 pg_epoch: 34 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c5261700 10 osd.0 pg_epoch: 34 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:34:11.615+0000 7fe2c5261700 10 osd.0 pg_epoch: 34 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c5261700 20 osd.0 pg_epoch: 34 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558e81a98000 2019-12-07T19:34:11.615+0000 7fe2c5261700 20 osd.0 pg_epoch: 34 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7fe2c5261700 20 osd.0 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7fe2c8a68700 10 osd.0 pg_epoch: 34 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2c5261700 10 osd.0 pg_epoch: 34 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 5 submitting 5 deferred done 0 stable 0 2019-12-07T19:34:11.615+0000 7fe2c5261700 10 osd.0 pg_epoch: 34 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:11.615+0000 7fe2c5261700 20 osd.0 pg_epoch: 34 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c5261700 10 osd.0 pg_epoch: 34 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.615+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:11.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2300 < min_flush_size 524288 2019-12-07T19:34:11.615+0000 7fe2cc26f700 20 osd.0 34 agent_entry empty queue 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:11.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3450 < min_flush_size 524288 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:11.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4600 < min_flush_size 524288 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:11.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5750 < min_flush_size 524288 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:11.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5773 < min_flush_size 524288 2019-12-07T19:34:11.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5773 < min_flush_size 524288 2019-12-07T19:34:11.615+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x14644d~168d to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.615+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x14644d 0x14644d~168d to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11644d 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x44d 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xada and padding block with 0x526 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x147ada 2019-12-07T19:34:11.615+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.615+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:11.615+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 5 cleaned 0 in 0.000132268s (0.000000112s flush + 0.000132156s kv commit) 2019-12-07T19:34:11.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8441af00,0x558e83e72c00,0x558e819a0900,0x558e83eb8900,0x558e819a0c00> 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441af00 kv_submitted 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8441af00 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441af00 finishing 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8441af00 onodes 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8441af00 done 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8441af00 [] 2019-12-07T19:34:11.615+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.615+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72c00 kv_submitted 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72c00 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72c00 finishing 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72c00 onodes 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72c00 done 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72c00 [] 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 kv_submitted 2019-12-07T19:34:11.615+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0900 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0900 finishing 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0900 onodes 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0900 done 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:11.615+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0900 [] 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8900 kv_submitted 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83eb8900 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8900 finishing 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83eb8900 onodes 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83eb8900 done 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83eb8900 [] 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 kv_submitted 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0c00 2019-12-07T19:34:11.615+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0c00 finishing 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0c00 onodes 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0c00 done 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:11.615+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0c00 [] 2019-12-07T19:34:11.615+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e34) pg 0x558e8193b000 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.as3 33 -> 34 2019-12-07T19:34:11.615+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 255 ==== MOSDECSubOpWrite(1.9s3 34/28 ECSubWrite(tid=157, reqid=osd.1.0:156, at_version=0'0, trim_to=0'0, roll_forward_to=34'17)) v2 ==== 909+0+0 (unknown 0 0 0) 0x558e845be000 con 0x558e818d7c00 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] handle_advance_map [1,4,2,0,5,3]/[1,4,2,0,5,3] -- 1/1 2019-12-07T19:34:11.615+0000 7fe2e7d3c700 15 osd.0 34 enqueue_op 0x558e819431e0 prio 127 cost 0 latency 0.000029 epoch 34 MOSDECSubOpWrite(1.9s3 34/28 ECSubWrite(tid=157, reqid=osd.1.0:156, at_version=0'0, trim_to=0'0, roll_forward_to=34'17)) v2 2019-12-07T19:34:11.615+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 34/28 ECSubWrite(tid=157, reqid=osd.1.0:156, at_version=0'0, trim_to=0'0, roll_forward_to=34'17)) v2) prio 127 cost 0 e34) 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 34/28 ECSubWrite(tid=157, reqid=osd.1.0:156, at_version=0'0, trim_to=0'0, roll_forward_to=34'17)) v2) prio 127 cost 0 e34) queued 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.615+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGOpItem(op=MOSDECSubOpWrite(1.9s3 34/28 ECSubWrite(tid=157, reqid=osd.1.0:156, at_version=0'0, trim_to=0'0, roll_forward_to=34'17)) v2) prio 127 cost 0 e34) pg 0x558e8192b000 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 34 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 34 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 osd.0 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 osd.0 34 dequeue_op 0x558e819431e0 prio 127 cost 0 latency 0.000063 MOSDECSubOpWrite(1.9s3 34/28 ECSubWrite(tid=157, reqid=osd.1.0:156, at_version=0'0, trim_to=0'0, roll_forward_to=34'17)) v2 pg pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] _handle_message: MOSDECSubOpWrite(1.9s3 34/28 ECSubWrite(tid=157, reqid=osd.1.0:156, at_version=0'0, trim_to=0'0, roll_forward_to=34'17)) v2 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 osd.0 pg_epoch: 34 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=33'15 active mbc={} ps=[4~1,6~2]] append_log log((0'0,34'17], crt=33'15) [] 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2]] rollforward: entry=34'16 (33'14) delete 1:9b8adabe:::smithi13140796-6:a by unknown.0.0:0 2019-12-07T19:34:11.613853+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e8417b500 seq 24 2019-12-07T19:34:11.615+0000 7fe2c7265700 20 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2]] rollforward: entry=34'17 (33'15) modify 1:9b8adabe:::smithi13140796-6:head by unknown.0.0:0 2019-12-07T19:34:11.613853+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:11.615+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417b500 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417b500 onodes shared_blobs 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2]] append_log approx pg log length = 17 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417b500 allocated 0x[] released 0x[] 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2]] append_log transaction_applied = 1 2019-12-07T19:34:11.615+0000 7fe2c7265700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 prepare 2019-12-07T19:34:11.615+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417b500 2019-12-07T19:34:11.615+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 io_done 2019-12-07T19:34:11.615+0000 7fe2c7265700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83eb8c00 seq 38 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558e8193b000 2019-12-07T19:34:11.616+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:a#10 onode 0x558e8402e500 txc 0x558e83eb8c00 2019-12-07T19:34:11.616+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:a#10 0x0 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore.blob(0x558e84755b20) put_ref 0x0~2000 Blob(0x558e84755b20 blob([0x3f000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e818667e0 loaded (sbid 0x2828 ref_map(0x3f000~2000=1)))) 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore.blob(0x558e84755b90) put_ref 0x0~2000 Blob(0x558e84755b90 blob([0x41000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e1c0 loaded (sbid 0x2829 ref_map(0x41000~2000=1)))) 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x147ada~495 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore.blob(0x558e83ac4ee0) put_ref 0x0~2000 Blob(0x558e83ac4ee0 blob([0x43000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81867960 loaded (sbid 0x282a ref_map(0x43000~2000=1)))) 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore.extentmap(0x558e8402e608) dirty_range mark inline shard dirty 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 NullEvt) prio 255 cost 10 e34) pg 0x558e81075000 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x147ada 0x147ada~495 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e84755b20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e818667e0 loaded (sbid 0x2828 ref_map(0x3f000~2000=1)))) 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x3f000~2000] 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x3f000~2000] from SharedBlob(0x558e818667e0 loaded (sbid 0x2828 ref_map())) 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x3f000~2000 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x117ada 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xada 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e84755b90 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8194e1c0 loaded (sbid 0x2829 ref_map(0x41000~2000=1)))) 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x41000~2000] 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x41000~2000] from SharedBlob(0x558e8194e1c0 loaded (sbid 0x2829 ref_map())) 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xf6f and padding block with 0x91 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x41000~2000 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e83ac4ee0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e81867960 loaded (sbid 0x282a ref_map(0x43000~2000=1)))) 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x43000~2000] 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [0x43000~2000] from SharedBlob(0x558e81867960 loaded (sbid 0x282a ref_map())) 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish release 0x43000~2000 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove gen and maybe_unshared_blobs 0x558e818667e0,0x558e81867960,0x558e8194e1c0 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.9s3_head 3#1:9b8adabe:::smithi13140796-6:a#10 = 0 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] handle_advance_map: 34 2019-12-07T19:34:11.616+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.2s0 33 -> 34 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 33 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83eb8c00 cost 5462 (1 ios * 4000 + 1462 bytes) 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83eb8c00 onodes shared_blobs 0x558e818667e0,0x558e81867960,0x558e8194e1c0 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2828 is empty 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x282a is empty 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2]] state: Active advmap 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2829 is empty 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83eb8c00 allocated 0x[] released 0x[3f000~6000] 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv release 0x3f000~6000 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 freelist release 0x3f000~6000 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] on_active_advmap new removed_snaps [a~1], snap_trimq now [a~1] 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8c00 prepare 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83eb8c00 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8c00 io_done 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 osd.0 34 dequeue_op 0x558e819431e0 finish 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] state: Started advmap 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] check_recovery_sources no source osds () went down 2019-12-07T19:34:11.616+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] plpg_on_pool_change 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] hit_set_clear 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] agent_stop 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 34 advance_pg new pool opts old pool opts 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] handle_activate_map: 34 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] activate_map 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] state: Active: handling ActMap 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] Active: kicking snap trim 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] kick_snap_trim: clean and snaps to trim, kicking 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x147f6f 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] exit NotTrimming 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000179287s (0.000000084s flush + 0.000179203s kv commit) 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] enter Trimming 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 145 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] enter Trimming/WaitReservation 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1320 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402e500 had 1 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417b500> 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 kv_submitted 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1343 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417b500 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1343 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7fe2caa6c700 15 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] publish_stats_to_osd 32:101 2019-12-07T19:34:11.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417b500 finishing 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417b500 onodes 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417b500 done 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:11.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417b500 [] 2019-12-07T19:34:11.616+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x147f6f~53f to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7fe2caa6c700 7 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] state: handle ActMap primary 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x147f6f 0x147f6f~53f to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x117f6f 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xf6f 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x4ae and padding block with 0xb52 2019-12-07T19:34:11.616+0000 7fe2caa6c700 15 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] publish_stats_to_osd 33:102 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] activate_map: Not dirtying info: last_persisted is 32 while current is 34 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 NullEvt 2019-12-07T19:34:11.616+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1484ae 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:11.616+0000 7fe2cea74700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: WaitReservation react SnapTrimReserved 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:11.616+0000 7fe2cea74700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: NotTrimming: trimming a 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000141442s (0.000000082s flush + 0.000141360s kv commit) 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:11.616+0000 7fe2cea74700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim_wait trimq=[a~1] ps=[4~1,6~2]] exit Trimming/WaitReservation 2019-12-07T19:34:11.616+0000 7fe2cea74700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:11.616+0000 7fe2cea74700 15 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] publish_stats_to_osd 34:103 2019-12-07T19:34:11.616+0000 7fe2cea74700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] enter Trimming/AwaitAsyncWork 2019-12-07T19:34:11.616+0000 7fe2cea74700 10 osd.0 34 queueing pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean trimq=[a~1] ps=[4~1,6~2]] for snaptrim 2019-12-07T19:34:11.616+0000 7fe2cea74700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=34) prio 5 cost 1048576 e34) 2019-12-07T19:34:11.616+0000 7fe2cea74700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2] 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:11.616+0000 7fe2cea74700 15 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] publish_stats_to_osd 34:104 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83eb8c00> 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8c00 kv_submitted 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83eb8c00 2019-12-07T19:34:11.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8c00 finishing 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83eb8c00 onodes 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83eb8c00 done 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:11.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83eb8c00 [3f000~6000] 2019-12-07T19:34:11.616+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release 0x3f000~6000 2019-12-07T19:34:11.616+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e8194e1c0) put 0x558e8194e1c0 removing self from set 0x558e819b0c10 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194e1d8 in 0x558e80219a40) _clear 2019-12-07T19:34:11.616+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.9s0 34/28 ECSubWriteReply(tid=157, last_complete=34'17, committed=1, applied=1)) v2 -- 0x558e83feec00 con 0x558e818d7c00 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 _rm buffer(0x558e81a47ef0 space 0x558e8194e1d8 0x0~1000 clean) 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 _rm buffer(0x558e84584000 space 0x558e8194e1d8 0x1000~1000 clean) 2019-12-07T19:34:11.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e81867960) put 0x558e81867960 removing self from set 0x558e819b0c10 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81867978 in 0x558e80219a40) _clear 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 _rm buffer(0x558e84584090 space 0x558e81867978 0x0~1000 clean) 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 _rm buffer(0x558e84584120 space 0x558e81867978 0x1000~1000 clean) 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore.sharedblob(0x558e818667e0) put 0x558e818667e0 removing self from set 0x558e819b0c10 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e818667f8 in 0x558e80219a40) _clear 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 _rm buffer(0x558e811713b0 space 0x558e818667f8 0x0~1000 clean) 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 _rm buffer(0x558e841ed830 space 0x558e818667f8 0x1000~1000 clean) 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:11.616+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 256 ==== pg_info2(1.9s3 1.9s3 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558e83ebbc00 con 0x558e818d7c00 2019-12-07T19:34:11.616+0000 7fe2e7d3c700 15 osd.0 34 enqueue_peering_evt 1.9s3 epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:11.616+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e34) pg 0x558e8192b000 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83eb8f00 seq 39 2019-12-07T19:34:11.616+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83eb8f00 cost 5289 (1 ios * 4000 + 1289 bytes) 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83eb8f00 onodes shared_blobs 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83eb8f00 allocated 0x[] released 0x[] 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8f00 prepare 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83eb8f00 2019-12-07T19:34:11.616+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8f00 io_done 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1126 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1149 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=34) prio 5 cost 1048576 e34) queued 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1484ae~47d to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGSnapTrim(pgid=1.2s0epoch_queued=34) prio 5 cost 1048576 e34) pg 0x558e81075000 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1484ae 0x1484ae~47d to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x1184ae 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x4ae 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x92b and padding block with 0x6d5 2019-12-07T19:34:11.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] snap_trimmer posting 2019-12-07T19:34:11.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: AwaitAsyncWork: trimming snap a 2019-12-07T19:34:11.616+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:11.616+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 1 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x14892b 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 bluestore.OmapIteratorImpl(0x558e819788c0) upper_bound after SNA_1_000000000000000A_.0_0000000000000001.2 key 0x00000000000000000000000000000402'.SNA_1_000000000000000A_.0_0000000000000001.2' 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000093024s (0.000000120s flush + 0.000092904s kv commit) 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 bluestore.OmapIteratorImpl(0x558e819788c0) valid is at 0x00000000000000000000000000000402'~' 2019-12-07T19:34:11.616+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 132 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558e83ebc300 con 0x558e818fe400 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 snap_mapper.get_next_objects_to_trim get_next(SNA_1_000000000000000A_.0_0000000000000001.2) returns -2 ,buffer::list(len=0, ) 2019-12-07T19:34:11.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim trimq=[a~1] ps=[4~1,6~2]] SnapTrimmer state: got ENOENT 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83eb8f00> 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8f00 kv_submitted 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83eb8f00 2019-12-07T19:34:11.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb8f00 finishing 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83eb8f00 onodes 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83eb8f00 done 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:11.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83eb8f00 [] 2019-12-07T19:34:11.616+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:11.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:11.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1,6~2]] SnapTrimmer state: adding snap a to purged_snaps 2019-12-07T19:34:11.616+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7fe2e6d3a700 15 osd.0 34 enqueue_peering_evt 1.bs1 epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:11.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1,6~2,a~1]] SnapTrimmer state: purged_snaps now [4~1,6~2,a~1], snap_trimq now [] 2019-12-07T19:34:11.616+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:11.616+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e845d4300 seq 41 2019-12-07T19:34:11.616+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:11.616+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:11.616+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.616+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:11.616+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e34) pg 0x558e83c78000 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:11.616+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:11.616+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845d4300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845d4300 onodes shared_blobs 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845d4300 allocated 0x[] released 0x[] 2019-12-07T19:34:11.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 34 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:11.616+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 133 ==== pg_info2(1.3s5 1.3s5 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558e83ebc300 con 0x558e818fe400 2019-12-07T19:34:11.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 34 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.616+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4300 prepare 2019-12-07T19:34:11.616+0000 7fe2c6a64700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845d4300 2019-12-07T19:34:11.616+0000 7fe2c6a64700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4300 io_done 2019-12-07T19:34:11.616+0000 7fe2e6d3a700 15 osd.0 34 enqueue_peering_evt 1.3s5 epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:11.616+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:11.616+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.616+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:11.616+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e819a0300 seq 17 2019-12-07T19:34:11.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1,6~2,a~1]] share_pg_info 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:11.616+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._biginfo' <- _biginfo 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.616+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._epoch' <- _epoch 2019-12-07T19:34:11.616+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 -- 0x558e8404ae00 con 0x558e818d7c00 2019-12-07T19:34:11.616+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:11.616+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a0300 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x14892b~495 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a0300 onodes shared_blobs 2019-12-07T19:34:11.616+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a0300 allocated 0x[] released 0x[] 2019-12-07T19:34:11.616+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x14892b 0x14892b~495 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11892b 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x92b 2019-12-07T19:34:11.616+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0300 prepare 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.616+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a0300 2019-12-07T19:34:11.616+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xdc0 and padding block with 0x240 2019-12-07T19:34:11.616+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0300 io_done 2019-12-07T19:34:11.616+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 -- 0x558e8404d800 con 0x558e818d7000 2019-12-07T19:34:11.616+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.616+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 -- 0x558e8404c300 con 0x558e818fe400 2019-12-07T19:34:11.616+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.616+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e34) pg 0x558e81231000 2019-12-07T19:34:11.616+0000 7fe2c6263700 10 osd.0 pg_epoch: 34 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:11.616+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 -- 0x558e8404bc00 con 0x558e818d6800 2019-12-07T19:34:11.616+0000 7fe2c6263700 10 osd.0 pg_epoch: 34 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.616+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:11.616+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 -- 0x558e8404d100 con 0x558e818fe000 2019-12-07T19:34:11.617+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:11.617+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e83d4cc00 seq 17 2019-12-07T19:34:11.617+0000 7fe2c6a64700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1,6~2,a~1]] exit Trimming/AwaitAsyncWork 2019-12-07T19:34:11.617+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:11.617+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._biginfo' <- _biginfo 2019-12-07T19:34:11.617+0000 7fe2c6a64700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean+snaptrim ps=[4~1,6~2,a~1]] exit Trimming 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x148dc0 2019-12-07T19:34:11.617+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._epoch' <- _epoch 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:11.617+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:11.617+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:11.617+0000 7fe2c6a64700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2,a~1] 2019-12-07T19:34:11.617+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4cc00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.617+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4cc00 onodes shared_blobs 2019-12-07T19:34:11.617+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4cc00 allocated 0x[] released 0x[] 2019-12-07T19:34:11.617+0000 7fe2c6a64700 15 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] publish_stats_to_osd 34:105 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000124877s (0.000000081s flush + 0.000124796s kv commit) 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.617+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 prepare 2019-12-07T19:34:11.617+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4cc00 2019-12-07T19:34:11.617+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 io_done 2019-12-07T19:34:11.617+0000 7fe2c6a64700 20 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] enter NotTrimming 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:11.617+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7fe2c6a64700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] SnapTrimmer state: NotTrimming react KickTrim 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:11.617+0000 7fe2c6a64700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] SnapTrimmer state: NotTrimming not clean or nothing to trim 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.617+0000 7fe2c6a64700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] snap_trimmer complete 2019-12-07T19:34:11.617+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x148dc0~495 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x148dc0 0x148dc0~495 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x118dc0 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xdc0 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x255 and padding block with 0xdab 2019-12-07T19:34:11.617+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e845d4300> 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4300 kv_submitted 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845d4300 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4300 finishing 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845d4300 onodes 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845d4300 done 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845d4300 [] 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:11.617+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 41 ==== pg_info2(1.4s1 1.4s1 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558e8404bc00 con 0x558e818fe000 2019-12-07T19:34:11.617+0000 7fe2e753b700 15 osd.0 34 enqueue_peering_evt 1.4s1 epoch_sent: 34 epoch_requested: 34 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s) 2019-12-07T19:34:11.617+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s)) prio 255 cost 10 e34) pg 0x558e81938000 2019-12-07T19:34:11.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 34 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.052553893s ub 44.052553893s int 16s) 2019-12-07T19:34:11.617+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 34 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x149255 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000252751s (0.000000073s flush + 0.000252678s kv commit) 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e8195f800 seq 14 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a0300> 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0300 kv_submitted 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a0300 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a0300 finishing 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a0300 onodes 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a0300 done 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a0300 [] 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.617+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195f800 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195f800 onodes shared_blobs 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195f800 allocated 0x[] released 0x[] 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 prepare 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195f800 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 io_done 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.617+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x149255~495 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x149255 0x149255~495 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x119255 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x255 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x6ea and padding block with 0x916 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1496ea 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000155886s (0.000000069s flush + 0.000155817s kv commit) 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1496ea~495 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1496ea 0x1496ea~495 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x1196ea 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x6ea 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb7f and padding block with 0x481 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83d4cc00> 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 kv_submitted 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4cc00 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4cc00 finishing 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4cc00 onodes 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4cc00 done 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4cc00 [] 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.617+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:11.617+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x149b7f 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:11.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000115128s (0.000000082s flush + 0.000115046s kv commit) 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:11.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195f800> 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 kv_submitted 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195f800 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f800 finishing 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195f800 onodes 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f800 done 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df900 q now empty 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195f800 [] 2019-12-07T19:34:11.617+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:11.617+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:11.617+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:11.628+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 0 Request: 832 2019-12-07T19:34:11.628+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 1 Request: 928 2019-12-07T19:34:11.628+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 2 Request: 0 2019-12-07T19:34:11.628+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 3 Request: 0 2019-12-07T19:34:11.628+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 4 Request: 0 2019-12-07T19:34:11.628+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 5 Request: 0 2019-12-07T19:34:11.628+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 6 Request: 0 2019-12-07T19:34:11.628+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 7 Request: 0 2019-12-07T19:34:11.628+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 8 Request: 0 2019-12-07T19:34:11.628+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 9 Request: 0 2019-12-07T19:34:11.628+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 10 Request: 0 2019-12-07T19:34:11.628+0000 7fe2dc28f700 10 rocksdb: request_cache_bytes Priority: 11 Request: 0 2019-12-07T19:34:11.628+0000 7fe2dc28f700 10 rocksdb: commit_cache_size old: 1073741824 new: 1073741824 2019-12-07T19:34:11.628+0000 7fe2dc28f700 10 rocksdb: commit_cache_size High Pri Pool Ratio set to 0.0258983 2019-12-07T19:34:11.628+0000 7fe2dc28f700 5 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.679+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.729+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.736+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 49 ==== pg_info2(1.ds5 1.ds5 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558e8404d100 con 0x558e818d6000 2019-12-07T19:34:11.736+0000 7fe2e753b700 15 osd.0 34 enqueue_peering_evt 1.ds5 epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:11.736+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.736+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.736+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.736+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.736+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s)) prio 255 cost 10 e34) pg 0x558e819e7000 2019-12-07T19:34:11.736+0000 7fe2c9269700 10 osd.0 pg_epoch: 34 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 44.090670001s ub 44.090670001s int 16s) 2019-12-07T19:34:11.736+0000 7fe2c9269700 10 osd.0 pg_epoch: 34 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.736+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:11.736+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:34:11.736+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e845d4f00 seq 21 2019-12-07T19:34:11.736+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:34:11.736+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:11.736+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:11.736+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:11.736+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:34:11.736+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845d4f00 cost 5307 (1 ios * 4000 + 1307 bytes) 2019-12-07T19:34:11.736+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845d4f00 onodes shared_blobs 2019-12-07T19:34:11.736+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845d4f00 allocated 0x[] released 0x[] 2019-12-07T19:34:11.736+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4f00 prepare 2019-12-07T19:34:11.736+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845d4f00 2019-12-07T19:34:11.736+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4f00 io_done 2019-12-07T19:34:11.736+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:11.736+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 50 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e34/34) v1 ==== 1092+0+0 (unknown 0 0 0) 0x558e8404d100 con 0x558e818d6000 2019-12-07T19:34:11.736+0000 7fe2e753b700 15 osd.0 34 enqueue_peering_evt 1.5s4 epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:11.736+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1150 < min_flush_size 524288 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1173 < min_flush_size 524288 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x149b7f~495 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x149b7f 0x149b7f~495 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x119b7f 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb7f 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x14 and padding block with 0xfec 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x14a014 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000140256s (0.000000148s flush + 0.000140108s kv commit) 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e845d4f00> 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.736+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4f00 kv_submitted 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845d4f00 2019-12-07T19:34:11.736+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d4f00 finishing 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845d4f00 onodes 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845d4f00 done 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:34:11.736+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845d4f00 [] 2019-12-07T19:34:11.736+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:11.736+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:11.736+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:11.736+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:11.736+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e34) pg 0x558e83d07000 2019-12-07T19:34:11.736+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:11.736+0000 7fe2c5261700 10 osd.0 pg_epoch: 34 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:11.736+0000 7fe2c5261700 10 osd.0 pg_epoch: 34 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2]] proc_primary_info updating purged_snaps to [4~1,6~2,a~1] 2019-12-07T19:34:11.736+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:11.736+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:11.736+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e84559200 seq 19 2019-12-07T19:34:11.736+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:11.736+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._biginfo' <- _biginfo 2019-12-07T19:34:11.736+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._epoch' <- _epoch 2019-12-07T19:34:11.736+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:11.736+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:11.736+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84559200 cost 5477 (1 ios * 4000 + 1477 bytes) 2019-12-07T19:34:11.736+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84559200 onodes shared_blobs 2019-12-07T19:34:11.736+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84559200 allocated 0x[] released 0x[] 2019-12-07T19:34:11.736+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559200 prepare 2019-12-07T19:34:11.736+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84559200 2019-12-07T19:34:11.736+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559200 io_done 2019-12-07T19:34:11.736+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1320 < min_flush_size 524288 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1343 < min_flush_size 524288 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1343 < min_flush_size 524288 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x14a014~53f to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x14a014 0x14a014~53f to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11a014 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x14 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x553 and padding block with 0xaad 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x14a553 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:11.736+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000118651s (0.000000103s flush + 0.000118548s kv commit) 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:11.736+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e84559200> 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:11.736+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559200 kv_submitted 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84559200 2019-12-07T19:34:11.736+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84559200 finishing 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84559200 onodes 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84559200 done 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:11.736+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:11.736+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84559200 [] 2019-12-07T19:34:11.736+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:11.736+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:11.736+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:11.779+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.829+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.879+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.929+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:11.979+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.029+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.079+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.129+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.180+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.230+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.243+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.2 172.21.15.131:0/37077 9 ==== osd_ping(ping e34 up_from 13 ping_stamp 2019-12-07T19:34:12.243674+0000/35.715393440s send_stamp 35.715393440s delta_ub 0.017206124s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e8182c000 2019-12-07T19:34:12.243+0000 7fe2e6d3a700 20 osd.0 34 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016893198s,0.017206124s]) 2019-12-07T19:34:12.243+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e34 up_from 13 ping_stamp 2019-12-07T19:34:12.243674+0000/35.715393440s send_stamp 35.698500242s delta_ub 18446744073.692658418s) v5 -- 0x558e845c2e00 con 0x558e8182c000 2019-12-07T19:34:12.243+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.2 172.21.15.131:0/37077 9 ==== osd_ping(ping e34 up_from 13 ping_stamp 2019-12-07T19:34:12.243674+0000/35.715393440s send_stamp 35.715393440s delta_ub 0.017206124s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84053400 con 0x558e8182d400 2019-12-07T19:34:12.243+0000 7fe2e6d3a700 10 osd.0 34 maybe_share_map con 0x558e818d6000 v2:172.21.15.131:6804/37077 map epoch 33 -> 34 (as per caller) 2019-12-07T19:34:12.243+0000 7fe2e7d3c700 20 osd.0 34 handle_osd_ping new stamps hbstamp(osd.2 up_from 13 peer_clock_delta [0.016829204s,0.017206124s]) 2019-12-07T19:34:12.243+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37077 -- osd_ping(ping_reply e34 up_from 13 ping_stamp 2019-12-07T19:34:12.243674+0000/35.715393440s send_stamp 35.698564236s delta_ub 18446744073.692722412s) v5 -- 0x558e84053a00 con 0x558e8182d400 2019-12-07T19:34:12.280+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.330+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.380+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.401+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:12.430+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.444+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83ddb800 0x558e83d4a100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:12.444+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83ddb800 0x558e83d4a100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:12.444+0000 7fe2e753b700 10 osd.0 34 ms_handle_authentication new session 0x558e83ff6680 con 0x558e83ddb800 entity client.admin addr 2019-12-07T19:34:12.444+0000 7fe2e753b700 10 osd.0 34 ms_handle_authentication session 0x558e83ff6680 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:12.444+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/3190856828 conn(0x558e83ddb800 0x558e83d4a100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=client.4705 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:12.444+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4705 172.21.15.131:0/3190856828 1 ==== command(tid 1: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558e8403eb40 con 0x558e83ddb800 2019-12-07T19:34:12.444+0000 7fe2d959f700 20 osd.0 34 OSD::ms_dispatch: command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:12.444+0000 7fe2d959f700 10 osd.0 34 do_waiters -- start 2019-12-07T19:34:12.444+0000 7fe2d959f700 10 osd.0 34 do_waiters -- finish 2019-12-07T19:34:12.444+0000 7fe2d959f700 20 osd.0 34 _dispatch 0x558e8403eb40 command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:12.444+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/3190856828 -- command_reply(tid 1: 0 ) v1 -- 0x558e83c82fc0 con 0x558e83ddb800 2019-12-07T19:34:12.465+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4705 172.21.15.131:0/3190856828 2 ==== command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 ==== 100+0+0 (crc 0 0 0) 0x558e8403e3c0 con 0x558e83ddb800 2019-12-07T19:34:12.465+0000 7fe2d959f700 20 osd.0 34 OSD::ms_dispatch: command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:12.465+0000 7fe2d959f700 10 osd.0 34 do_waiters -- start 2019-12-07T19:34:12.465+0000 7fe2d959f700 10 osd.0 34 do_waiters -- finish 2019-12-07T19:34:12.465+0000 7fe2d959f700 20 osd.0 34 _dispatch 0x558e8403e3c0 command(tid 2: {"injected_args": ["--osd_enable_op_tracker=false"], "prefix": "injectargs"}) v1 2019-12-07T19:34:12.467+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/3190856828 -- command_reply(tid 2: 0 osd_enable_op_tracker = 'false' ) v1 -- 0x558e8022f180 con 0x558e83ddb800 2019-12-07T19:34:12.468+0000 7fe2e2ca2700 10 osd.0 34 tick_without_osd_lock 2019-12-07T19:34:12.468+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fa83000/0x40000000/0x1680000000, data 0x54398b/0x57b000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:12.468+0000 7fe2e2ca2700 25 osd.0 34 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955408+0000 last_rx_front 2019-12-07T19:34:10.955347+0000 2019-12-07T19:34:12.468+0000 7fe2e2ca2700 25 osd.0 34 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955302+0000 last_rx_front 2019-12-07T19:34:10.955288+0000 2019-12-07T19:34:12.468+0000 7fe2e2ca2700 25 osd.0 34 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955335+0000 last_rx_front 2019-12-07T19:34:10.955354+0000 2019-12-07T19:34:12.468+0000 7fe2e2ca2700 25 osd.0 34 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955428+0000 last_rx_front 2019-12-07T19:34:10.955392+0000 2019-12-07T19:34:12.468+0000 7fe2e2ca2700 25 osd.0 34 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955481+0000 last_rx_front 2019-12-07T19:34:10.955444+0000 2019-12-07T19:34:12.468+0000 7fe2e2ca2700 25 osd.0 34 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955373+0000 last_rx_front 2019-12-07T19:34:10.955455+0000 2019-12-07T19:34:12.468+0000 7fe2e2ca2700 20 osd.0 34 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:12.468+0000 7fe2e2ca2700 10 osd.0 34 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:12.468+0000 7fe2e2ca2700 20 osd.0 34 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:12.468+0000 7fe2e2ca2700 10 osd.0 34 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:12.480+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.530+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.561+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.4 172.21.15.109:0/36938 6 ==== osd_ping(ping e34 up_from 13 ping_stamp 2019-12-07T19:34:12.561802+0000/36.011028212s send_stamp 36.011028212s delta_ub 18446744073.704155227s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818ee800 con 0x558e81985400 2019-12-07T19:34:12.561+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.4 172.21.15.109:0/36938 6 ==== osd_ping(ping e34 up_from 13 ping_stamp 2019-12-07T19:34:12.561802+0000/36.011028212s send_stamp 36.011028212s delta_ub 18446744073.704155227s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83fee000 con 0x558e81985800 2019-12-07T19:34:12.561+0000 7fe2e6d3a700 20 osd.0 34 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703838427s,18446744073.704155227s]) 2019-12-07T19:34:12.561+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e34 up_from 13 ping_stamp 2019-12-07T19:34:12.561802+0000/36.011028212s send_stamp 36.016741401s delta_ub 0.005713189s) v5 -- 0x558e845c2e00 con 0x558e81985400 2019-12-07T19:34:12.561+0000 7fe2e6d3a700 10 osd.0 34 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 33 -> 34 (as per caller) 2019-12-07T19:34:12.561+0000 7fe2e7d3c700 20 osd.0 34 handle_osd_ping new stamps hbstamp(osd.4 up_from 13 peer_clock_delta [18446744073.703802869s,18446744073.704155227s]) 2019-12-07T19:34:12.561+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36938 -- osd_ping(ping_reply e34 up_from 13 ping_stamp 2019-12-07T19:34:12.561802+0000/36.011028212s send_stamp 36.016776959s delta_ub 0.005748747s) v5 -- 0x558e84053a00 con 0x558e81985800 2019-12-07T19:34:12.580+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 608479 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.584+0000 7fe2e4535700 10 osd.0 34 tick 2019-12-07T19:34:12.584+0000 7fe2e4535700 10 osd.0 34 do_waiters -- start 2019-12-07T19:34:12.584+0000 7fe2e4535700 10 osd.0 34 do_waiters -- finish 2019-12-07T19:34:12.584+0000 7fe2e4535700 20 osd.0 34 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:12.609+0000 7fe2cca70700 15 osd.0 34 enqueue_peering_evt 1.2s0 epoch_sent: 26 epoch_requested: 26 RenewLease 2019-12-07T19:34:12.609+0000 7fe2cca70700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 RenewLease) prio 255 cost 10 e26) 2019-12-07T19:34:12.609+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.609+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 RenewLease) prio 255 cost 10 e26) queued 2019-12-07T19:34:12.609+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.609+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 26 epoch_requested: 26 RenewLease) prio 255 cost 10 e26) pg 0x558e81075000 2019-12-07T19:34:12.609+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 26 epoch_requested: 26 RenewLease 2019-12-07T19:34:12.609+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] old_peering_msg reply_epoch 26 query_epoch 26 last_peering_reset 28 2019-12-07T19:34:12.610+0000 7fe2caa6c700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] discard old epoch_sent: 26 epoch_requested: 26 RenewLease 2019-12-07T19:34:12.610+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:12.610+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:12.611+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 28 ==== osd_map(34..35 src has 1..35) v4 ==== 1258+0+0 (crc 0 0 0) 0x558e81956b40 con 0x558e80f39000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 osd.0 34 OSD::ms_dispatch: osd_map(34..35 src has 1..35) v4 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 osd.0 34 do_waiters -- start 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 osd.0 34 do_waiters -- finish 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 osd.0 34 _dispatch 0x558e81956b40 osd_map(34..35 src has 1..35) v4 2019-12-07T19:34:12.611+0000 7fe2d959f700 3 osd.0 34 handle_osd_map epochs [34,35], i have 34, src has [1,35] 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 osd.0 34 handle_osd_map got inc map for epoch 35 2019-12-07T19:34:12.611+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3ff8e826:::osdmap.34:0# 0x0~0 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1464 size 0x1464 (5220) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83e6c690 blob([0x4ba000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e84755180 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84755110 blob([!~1000,0x4bb000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,464]) SharedBlob(0x558e847550a0 sbid 0x0)) need 0x1000~464 cache has 0x[1000~464] 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3ff8e826:::osdmap.34:0# 0x0~1464 = 5220 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 osd.0 34 add_map_bl 34 5220 bytes 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 osd.0 34 got_full_map 35, nothing requested 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 osd.0 34 handle_osd_map pg_num_history pg_num_history(e35 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps purged_snaps {35={}} 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 osd.0 34 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e35 [1,35] lci=[13,35]) 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e819a1500 seq 27 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:cdeb972f:::inc_osdmap.35:0# key 0x7f7fffffffffffffffcdeb97'/!inc_osdmap.35!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:12.611+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:cdeb972f:::inc_osdmap.35:0# 0x0~24d 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1162 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:cdeb972f:::inc_osdmap.35:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24d 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a1500 1 blobs 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x3f000~1000/1000,1000,0 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x3f000~1000] 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e841905b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f95730 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e841905b0 blob([0x3f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f95730 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.blob(0x558e841905b0) get_ref 0x0~24d Blob(0x558e841905b0 blob([0x3f000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f95730 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.blob(0x558e841905b0) get_ref init 0x1000, 1000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558e841905b0 blob([0x3f000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558e83f95730 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e83f95748 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x24d 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f29788) dirty_range mark inline shard dirty 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:cdeb972f:::inc_osdmap.35:0# 0x0~24d = 0 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3138e826:::osdmap.35:0# key 0x7f7fffffffffffffff3138e8'&!osdmap.35!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:12.611+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3138e826:::osdmap.35:0# 0x0~1448 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1163 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3138e826:::osdmap.35:0# 0x0~1448 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~448 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbb8 on front/back, now 0x0~1000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a1500 2 blobs 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x40000~2000/1000,2000,0 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x40000~2000] 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83f496c0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83f94620 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83f496c0 blob([0x40000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f94620 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.blob(0x558e83f496c0) get_ref 0x0~1000 Blob(0x558e83f496c0 blob([0x40000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83f94620 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.blob(0x558e83f496c0) get_ref init 0x1000, 1000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e83f496c0 blob([0x40000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f94620 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e83f94638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84549c70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84190230 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84549c70 blob([!~1000,0x41000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84190230 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.blob(0x558e84549c70) get_ref 0x1000~448 Blob(0x558e84549c70 blob([!~1000,0x41000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e84190230 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.blob(0x558e84549c70) get_ref init 0x2000, 1000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~448: 0x1000~448 Blob(0x558e84549c70 blob([!~1000,0x41000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558e84190230 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e84190248 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1448 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.extentmap(0x558e84771008) dirty_range mark inline shard dirty 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3138e826:::osdmap.35:0# 0x0~1448 = 0 2019-12-07T19:34:12.611+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a1500 0 blobs 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:12.611+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:12.611+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:12.611+0000 7fe2d959f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e819a1500 0 blobs 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e819a1500 cost 23262 (4 ios * 4000 + 7262 bytes) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e819a1500 onodes 0x558e80251b80,0x558e817b9180,0x558e83f29680,0x558e84770f00 shared_blobs 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f29788) update #-1:cdeb972f:::inc_osdmap.35:0# 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.extentmap(0x558e83f29788) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:cdeb972f:::inc_osdmap.35:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.extentmap(0x558e84771008) update #-1:3138e826:::osdmap.35:0# 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore.extentmap(0x558e84771008) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3138e826:::osdmap.35:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e819a1500 allocated 0x[3f000~3000] released 0x[] 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 freelist allocate 0x3f000~3000 2019-12-07T19:34:12.611+0000 7fe2d959f700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 prepare 2019-12-07T19:34:12.611+0000 7fe2d959f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e819a1500 2019-12-07T19:34:12.612+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 aio_wait 2019-12-07T19:34:12.612+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e819a1500 2019-12-07T19:34:12.612+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 io_done 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:12.612+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83f29680 had 1 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84770f00 had 1 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:12.612+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:12.612+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:12.612+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:12.612+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x14a553~226c to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:12.612+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x14a553 0x14a553~226c to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11a553 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x553 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x7bf and padding block with 0x841 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x14c7bf 2019-12-07T19:34:12.612+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:12.612+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:12.612+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000137796s (0.000004456s flush + 0.000133340s kv commit) 2019-12-07T19:34:12.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:12.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:12.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e819a1500> 2019-12-07T19:34:12.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:12.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 kv_submitted 2019-12-07T19:34:12.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e819a1500 2019-12-07T19:34:12.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e819a1500 osr 0x558e8107a500 2019-12-07T19:34:12.612+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e845eedc0) _discard drop 24 0x1b000~1000 2019-12-07T19:34:12.612+0000 7fe2d7285700 10 osd.0 34 _committed_osd_maps 35..35 2019-12-07T19:34:12.612+0000 7fe2d7285700 10 osd.0 34 advance to epoch 35 (<= last 35 <= newest_map 35) 2019-12-07T19:34:12.612+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e845eedc0) prepare_write seq 25 0x1b000~1000 crc a2e3f30a 2019-12-07T19:34:12.612+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e845eedc0) _discard drop 24 0x2000~1000 2019-12-07T19:34:12.612+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e845eedc0) prepare_write seq 25 0x2000~1000 crc ae140242 2019-12-07T19:34:12.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:12.612+0000 7fe2d7285700 7 osd.0 35 consume_map version 35 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 35 identify_splits_and_merges 1.8s5 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 35 identify_splits_and_merges 1.9s3 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 35 identify_splits_and_merges 1.1s1 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 35 identify_splits_and_merges 1.as3 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 35 identify_splits_and_merges 1.2s0 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 35 identify_splits_and_merges 1.3s5 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 35 identify_splits_and_merges 1.bs1 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 35 identify_splits_and_merges 1.4s1 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 35 identify_splits_and_merges 1.5s4 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 35 identify_splits_and_merges 1.ds5 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 35 identify_splits_and_merges 1.es4 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 35 identify_splits_and_merges 1.7s3 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 35 identify_splits_and_merges 1.fs1 e34 to e35 pg_nums {18=16} 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 35 prune_pg_created 2019-12-07T19:34:12.612+0000 7fe2d7285700 10 osd.0:0.consume_map 35 (was 34) 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 2019-12-07T19:34:12.612+0000 7fe2d7285700 10 osd.0:1.consume_map 35 (was 34) 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:12.612+0000 7fe2d7285700 10 osd.0:2.consume_map 35 (was 34) 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:12.612+0000 7fe2d7285700 10 osd.0:3.consume_map 35 (was 34) 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 2019-12-07T19:34:12.612+0000 7fe2d7285700 10 osd.0:4.consume_map 35 (was 34) 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:12.612+0000 7fe2d7285700 10 osd.0:5.consume_map 35 (was 34) 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:12.612+0000 7fe2d7285700 10 osd.0:6.consume_map 35 (was 34) 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:12.612+0000 7fe2d7285700 10 osd.0:7.consume_map 35 (was 34) 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:12.612+0000 7fe2d7285700 10 osd.0 35 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:12.612+0000 7fe2d7285700 15 osd.0 35 enqueue_peering_evt 1.8s5 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.612+0000 7fe2d7285700 15 osd.0 35 enqueue_peering_evt 1.9s3 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.612+0000 7fe2d7285700 15 osd.0 35 enqueue_peering_evt 1.1s1 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.612+0000 7fe2d7285700 15 osd.0 35 enqueue_peering_evt 1.as3 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.612+0000 7fe2d7285700 15 osd.0 35 enqueue_peering_evt 1.2s0 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.612+0000 7fe2d7285700 15 osd.0 35 enqueue_peering_evt 1.3s5 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.612+0000 7fe2d7285700 15 osd.0 35 enqueue_peering_evt 1.bs1 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.612+0000 7fe2d7285700 15 osd.0 35 enqueue_peering_evt 1.4s1 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.612+0000 7fe2d7285700 15 osd.0 35 enqueue_peering_evt 1.5s4 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.612+0000 7fe2d7285700 15 osd.0 35 enqueue_peering_evt 1.ds5 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.612+0000 7fe2d7285700 15 osd.0 35 enqueue_peering_evt 1.es4 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.612+0000 7fe2d7285700 15 osd.0 35 enqueue_peering_evt 1.7s3 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.612+0000 7fe2d7285700 15 osd.0 35 enqueue_peering_evt 1.fs1 epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) 2019-12-07T19:34:12.612+0000 7fe2d7285700 7 osd.0 35 activate_map version 35 2019-12-07T19:34:12.612+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.612+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558e81938000 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558e8193b000 2019-12-07T19:34:12.612+0000 7fe2c5a62700 10 osd.0 pg_epoch: 34 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 34 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.as3 34 -> 35 2019-12-07T19:34:12.612+0000 7fe2c5a62700 20 osd.0:4.update_pg_epoch 1.4s1 34 -> 35 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 34 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,4,2,0,5,3]/[1,4,2,0,5,3] -- 1/1 2019-12-07T19:34:12.612+0000 7fe2c5a62700 10 osd.0 pg_epoch: 34 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.612+0000 7fe2c5a62700 10 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558e81231000 2019-12-07T19:34:12.612+0000 7fe2c5a62700 10 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.612+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:34:12.612+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.612+0000 7fe2c5a62700 10 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 pg_epoch: 35 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 34 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.612+0000 7fe2c5a62700 20 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0:3.update_pg_epoch 1.3s5 34 -> 35 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 pg_epoch: 35 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.612+0000 7fe2c5a62700 20 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.612+0000 7fe2c5a62700 20 osd.0 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 34 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.612+0000 7fe2c5a62700 10 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 pg_epoch: 35 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 35 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2c5a62700 10 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 35 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.3s5_head options 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.612+0000 7fe2c5a62700 20 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558e81075000 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 35 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.612+0000 7fe2c5a62700 10 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 pg_epoch: 35 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.612+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 pg_epoch: 35 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.2s0 34 -> 35 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 34 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:12.612+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 35 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.612+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] state: Active advmap 2019-12-07T19:34:12.612+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558e81a98000 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 35 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 pg_epoch: 35 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.612+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 35 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c4a60700 10 osd.0 pg_epoch: 34 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.612+0000 7fe2c4a60700 20 osd.0:6.update_pg_epoch 1.es4 34 -> 35 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558e83c78000 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.612+0000 7fe2c4a60700 10 osd.0 pg_epoch: 34 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,3,4,5,0,2]/[1,3,4,5,0,2] -- 1/1 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] state: Active: handling ActMap 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] Active: kicking snap trim 2019-12-07T19:34:12.612+0000 7fe2c4a60700 10 osd.0 pg_epoch: 35 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.612+0000 7fe2c6a64700 7 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] state: handle ActMap primary 2019-12-07T19:34:12.612+0000 7fe2c4a60700 10 osd.0 pg_epoch: 35 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] prepare_stats_for_publish reporting purged_snaps [4~1,6~2,a~1] 2019-12-07T19:34:12.612+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 34 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0:3.update_pg_epoch 1.bs1 34 -> 35 2019-12-07T19:34:12.612+0000 7fe2c6a64700 15 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] publish_stats_to_osd 34:106 2019-12-07T19:34:12.612+0000 7fe2c4a60700 10 osd.0 pg_epoch: 35 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 34 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:12.612+0000 7fe2c4a60700 20 osd.0 pg_epoch: 35 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c4a60700 20 osd.0 pg_epoch: 35 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.612+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 35 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.612+0000 7fe2c4a60700 20 osd.0 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 35 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.612+0000 7fe2c4a60700 10 osd.0 pg_epoch: 35 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.612+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs1_head options 2019-12-07T19:34:12.612+0000 7fe2c4a60700 10 osd.0 pg_epoch: 35 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 35 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.612+0000 7fe2c4a60700 20 osd.0 pg_epoch: 35 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.612+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 pg_epoch: 35 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.612+0000 7fe2c4a60700 10 osd.0 pg_epoch: 35 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 pg_epoch: 35 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.612+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.612+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 35 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 35 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 pg_epoch: 35 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558e8122e000 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 osd.0 pg_epoch: 35 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.7s3 34 -> 35 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 35 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 35 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.612+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 35 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 pg_epoch: 35 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 pg_epoch: 35 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 35 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 35 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 pg_epoch: 35 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 35 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558e81a9b000 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.fs1 34 -> 35 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 34 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [3,0,5,4,1,2]/[3,0,5,4,1,2] -- 3/3 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 35 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 35 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.612+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 35 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 pg_epoch: 35 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 pg_epoch: 35 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 35 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 35 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 pg_epoch: 35 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 osd.0 pg_epoch: 35 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.612+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:12.612+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:12.612+0000 7fe2cc26f700 20 osd.0 35 agent_entry empty queue 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.613+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.613+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558e8192b000 2019-12-07T19:34:12.613+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.613+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558e841a4000 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.9s3 34 -> 35 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 34 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:12.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 34 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.613+0000 7fe2cba6e700 20 osd.0:0.update_pg_epoch 1.8s5 34 -> 35 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 35 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 34 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,4,5,2,3,0]/[1,4,5,2,3,0] -- 1/1 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 35 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.613+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:34:12.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 35 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 35 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 35 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.613+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.8s5_head options 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 pg_epoch: 35 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 35 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 pg_epoch: 35 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.613+0000 7fe2cba6e700 20 osd.0 pg_epoch: 35 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 35 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.613+0000 7fe2cba6e700 20 osd.0 pg_epoch: 35 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.613+0000 7fe2cba6e700 20 osd.0 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 35 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 35 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 pg_epoch: 35 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:12.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 35 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 35 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.613+0000 7fe2cba6e700 20 osd.0 pg_epoch: 35 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.613+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558e819e4000 2019-12-07T19:34:12.613+0000 7fe2cba6e700 10 osd.0 pg_epoch: 35 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.613+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:12.613+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 34 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.1s1 34 -> 35 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 34 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [3,0,2,4,5,1]/[3,0,2,4,5,1] -- 3/3 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 35 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 35 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.613+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 35 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 pg_epoch: 35 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 pg_epoch: 35 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 35 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 35 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 pg_epoch: 35 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 osd.0 pg_epoch: 35 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.613+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:12.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558e83d07000 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 34 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.5s4 34 -> 35 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 34 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [2,1,3,4,0,5]/[2,1,3,4,0,5] -- 2/2 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.613+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.5s4_head options 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 pg_epoch: 35 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 pg_epoch: 35 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 pg_epoch: 35 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) queued 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 35 epoch_requested: 35 NullEvt) prio 255 cost 10 e35) pg 0x558e819e7000 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 34 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 35 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.ds5 34 -> 35 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 34 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:12.613+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 35 advance_pg new pool opts old pool opts 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 35 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 35 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 35 epoch_requested: 35 NullEvt 2019-12-07T19:34:12.613+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:12.613+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:12.616+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 42 ==== pg_lease(1.4s1 pg_lease(ru 44.052553893s ub 52.052753396s int 16s) e34/34) v1 ==== 58+0+0 (unknown 0 0 0) 0x558e845c3400 con 0x558e818fe000 2019-12-07T19:34:12.616+0000 7fe2e753b700 15 osd.0 35 enqueue_peering_evt 1.4s1 epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.7 pg_lease(ru 44.052553893s ub 52.052753396s int 16s) 2019-12-07T19:34:12.616+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.7 pg_lease(ru 44.052553893s ub 52.052753396s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:12.616+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.616+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.7 pg_lease(ru 44.052553893s ub 52.052753396s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:12.616+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.616+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.7 pg_lease(ru 44.052553893s ub 52.052753396s int 16s)) prio 255 cost 10 e34) pg 0x558e81938000 2019-12-07T19:34:12.616+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.7 pg_lease(ru 44.052553893s ub 52.052753396s int 16s) 2019-12-07T19:34:12.616+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] proc_lease pg_lease(ru 44.052553893s ub 52.052753396s int 16s) 2019-12-07T19:34:12.616+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] peer_clock_delta_ub 18446744073.690493873s -> ru 44.071611636s 2019-12-07T19:34:12.616+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] proc_lease readable_until now 44.071611636s 2019-12-07T19:34:12.616+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] peer_clock_delta_lb 18446744073.689995278s -> ruub 52.072309734s 2019-12-07T19:34:12.616+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] proc_lease readable_until_ub now 52.072309734s 2019-12-07T19:34:12.616+0000 7fe2c9a6a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_lease_ack(1.4s0 pg_lease_ack(ruub 52.052753396s) e35/35) v1 -- 0x558e817b4d20 con 0x558e818fe000 2019-12-07T19:34:12.616+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:12.616+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:12.618+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 51 ==== pg_lease(1.ds5 pg_lease(ru 44.090670001s ub 52.090950317s int 16s) e34/34) v1 ==== 58+0+0 (unknown 0 0 0) 0x558e845c3400 con 0x558e818d6000 2019-12-07T19:34:12.618+0000 7fe2e753b700 15 osd.0 35 enqueue_peering_evt 1.ds5 epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.2 pg_lease(ru 44.090670001s ub 52.090950317s int 16s) 2019-12-07T19:34:12.618+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.2 pg_lease(ru 44.090670001s ub 52.090950317s int 16s)) prio 255 cost 10 e34) 2019-12-07T19:34:12.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:12.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.2 pg_lease(ru 44.090670001s ub 52.090950317s int 16s)) prio 255 cost 10 e34) queued 2019-12-07T19:34:12.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:12.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.2 pg_lease(ru 44.090670001s ub 52.090950317s int 16s)) prio 255 cost 10 e34) pg 0x558e819e7000 2019-12-07T19:34:12.618+0000 7fe2c5261700 10 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 34 epoch_requested: 34 MLease epoch 34 from osd.2 pg_lease(ru 44.090670001s ub 52.090950317s int 16s) 2019-12-07T19:34:12.618+0000 7fe2c5261700 10 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] proc_lease pg_lease(ru 44.090670001s ub 52.090950317s int 16s) 2019-12-07T19:34:12.618+0000 7fe2c5261700 20 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] peer_clock_delta_ub 0.017206124s -> ru 44.073463877s 2019-12-07T19:34:12.618+0000 7fe2c5261700 20 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] proc_lease readable_until now 44.073463877s 2019-12-07T19:34:12.618+0000 7fe2c5261700 20 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] peer_clock_delta_lb 0.016829204s -> ruub 52.074121113s 2019-12-07T19:34:12.618+0000 7fe2c5261700 20 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] proc_lease readable_until_ub now 52.074121113s 2019-12-07T19:34:12.618+0000 7fe2c5261700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease_ack(1.ds0 pg_lease_ack(ruub 52.090950317s) e35/35) v1 -- 0x558e8122dc20 con 0x558e818d6000 2019-12-07T19:34:12.618+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:12.618+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:12.626+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/3190856828 conn(0x558e83ddb800 msgr2=0x558e83d4a100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 61 2019-12-07T19:34:12.626+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/3190856828 conn(0x558e83ddb800 msgr2=0x558e83d4a100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2019-12-07T19:34:12.626+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/3190856828 conn(0x558e83ddb800 0x558e83d4a100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).handle_read_frame_preamble_main read frame length and tag failed r=-1 ((1) Operation not permitted) 2019-12-07T19:34:12.626+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/3190856828 conn(0x558e83ddb800 0x558e83d4a100 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).stop 2019-12-07T19:34:12.626+0000 7fe2d959f700 2 osd.0 35 ms_handle_reset con 0x558e83ddb800 session 0x558e83ff6680 2019-12-07T19:34:12.630+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) deferred_try_submit 1 osrs, 4 txcs 2019-12-07T19:34:12.630+0000 7fe2dc28f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock osr 0x558e8107a500 2 ios pending 2019-12-07T19:34:12.630+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 25 0x2000~1000 2019-12-07T19:34:12.630+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x2000~1000 crc ae140242 2019-12-07T19:34:12.630+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock seq 25 0x1b000~1000 2019-12-07T19:34:12.630+0000 7fe2dc28f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_submit_unlock write 0x1b000~1000 crc a2e3f30a 2019-12-07T19:34:12.631+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.631+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish osr 0x558e8107a500 2019-12-07T19:34:12.631+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_aio_finish dequeueing 2019-12-07T19:34:12.681+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.731+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.731+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:12.781+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.831+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.856+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:12.881+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.931+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:12.981+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.031+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.081+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.131+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.158+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:13.182+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.232+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.282+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.332+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.382+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.428+0000 7fe2e2ca2700 10 osd.0 35 tick_without_osd_lock 2019-12-07T19:34:13.428+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fa80000/0x40000000/0x1680000000, data 0x545020/0x57e000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:13.428+0000 7fe2e2ca2700 25 osd.0 35 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955408+0000 last_rx_front 2019-12-07T19:34:10.955347+0000 2019-12-07T19:34:13.428+0000 7fe2e2ca2700 25 osd.0 35 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955302+0000 last_rx_front 2019-12-07T19:34:10.955288+0000 2019-12-07T19:34:13.428+0000 7fe2e2ca2700 25 osd.0 35 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955335+0000 last_rx_front 2019-12-07T19:34:10.955354+0000 2019-12-07T19:34:13.428+0000 7fe2e2ca2700 25 osd.0 35 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955428+0000 last_rx_front 2019-12-07T19:34:10.955392+0000 2019-12-07T19:34:13.428+0000 7fe2e2ca2700 25 osd.0 35 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955481+0000 last_rx_front 2019-12-07T19:34:10.955444+0000 2019-12-07T19:34:13.428+0000 7fe2e2ca2700 25 osd.0 35 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955373+0000 last_rx_front 2019-12-07T19:34:10.955455+0000 2019-12-07T19:34:13.428+0000 7fe2e2ca2700 20 osd.0 35 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:13.428+0000 7fe2e2ca2700 10 osd.0 35 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:13.428+0000 7fe2e2ca2700 20 osd.0 35 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:13.428+0000 7fe2e2ca2700 10 osd.0 35 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:13.432+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.454+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.1 172.21.15.131:0/37078 5 ==== osd_ping(ping e35 up_from 13 ping_stamp 2019-12-07T19:34:13.454703+0000/36.911626419s send_stamp 36.911626419s delta_ub 0.002418470s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e84053400 con 0x558e81985000 2019-12-07T19:34:13.454+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.1 172.21.15.131:0/37078 5 ==== osd_ping(ping e35 up_from 13 ping_stamp 2019-12-07T19:34:13.454703+0000/36.911626419s send_stamp 36.911626419s delta_ub 0.002418470s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e81984c00 2019-12-07T19:34:13.454+0000 7fe2e6d3a700 20 osd.0 35 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002087727s,0.002418470s]) 2019-12-07T19:34:13.454+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e35 up_from 13 ping_stamp 2019-12-07T19:34:13.454703+0000/36.911626419s send_stamp 36.909538692s delta_ub 18446744073.707463889s) v5 -- 0x558e845c2e00 con 0x558e81984c00 2019-12-07T19:34:13.454+0000 7fe2e6d3a700 10 osd.0 35 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 34 -> 35 (as per caller) 2019-12-07T19:34:13.454+0000 7fe2e7d3c700 20 osd.0 35 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002046128s,0.002418470s]) 2019-12-07T19:34:13.454+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e35 up_from 13 ping_stamp 2019-12-07T19:34:13.454703+0000/36.911626419s send_stamp 36.909580291s delta_ub 18446744073.707505488s) v5 -- 0x558e84053a00 con 0x558e81985000 2019-12-07T19:34:13.482+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.533+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.572+0000 7fe2e4535700 10 osd.0 35 tick 2019-12-07T19:34:13.572+0000 7fe2e4535700 10 osd.0 35 do_waiters -- start 2019-12-07T19:34:13.572+0000 7fe2e4535700 10 osd.0 35 do_waiters -- finish 2019-12-07T19:34:13.572+0000 7fe2e4535700 20 osd.0 35 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:13.582+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 611715 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 257 ==== pg_info2(1.as3 1.as3 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d800 con 0x558e818d7c00 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 15 osd.0 35 enqueue_peering_evt 1.as3 epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 258 ==== pg_info2(1.9s3 1.9s3 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d800 con 0x558e818d7c00 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 15 osd.0 35 enqueue_peering_evt 1.9s3 epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 259 ==== pg_info2(1.7s3 1.7s3 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d800 con 0x558e818d7c00 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 15 osd.0 35 enqueue_peering_evt 1.7s3 epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 260 ==== pg_info2(1.8s5 1.8s5 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d800 con 0x558e818d7c00 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 15 osd.0 35 enqueue_peering_evt 1.8s5 epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.604+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e36) pg 0x558e841a4000 2019-12-07T19:34:13.604+0000 7fe2c7a66700 20 osd.0 op_wq _add_slot_waiter 1.8s5 peering, item epoch is 36, will wait on OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.604+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e36) pg 0x558e8193b000 2019-12-07T19:34:13.604+0000 7fe2caa6c700 20 osd.0 op_wq _add_slot_waiter 1.as3 peering, item epoch is 36, will wait on OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.604+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e36) pg 0x558e8192b000 2019-12-07T19:34:13.604+0000 7fe2c7265700 20 osd.0 op_wq _add_slot_waiter 1.9s3 peering, item epoch is 36, will wait on OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.604+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e36) pg 0x558e8122e000 2019-12-07T19:34:13.604+0000 7fe2c425f700 20 osd.0 op_wq _add_slot_waiter 1.7s3 peering, item epoch is 36, will wait on OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 261 ==== pg_info2(1.es4 1.es4 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d800 con 0x558e818d7c00 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 15 osd.0 35 enqueue_peering_evt 1.es4 epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:13.604+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.604+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.604+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e36) pg 0x558e81a98000 2019-12-07T19:34:13.604+0000 7fe2c8a68700 20 osd.0 op_wq _add_slot_waiter 1.es4 peering, item epoch is 36, will wait on OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.604+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:13.605+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 43 ==== pg_info2(1.4s1 1.4s1 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d100 con 0x558e818fe000 2019-12-07T19:34:13.605+0000 7fe2e753b700 15 osd.0 35 enqueue_peering_evt 1.4s1 epoch_sent: 36 epoch_requested: 36 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.052753396s ub 52.052753396s int 16s) 2019-12-07T19:34:13.605+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.052753396s ub 52.052753396s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.605+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.605+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.052753396s ub 52.052753396s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.605+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.605+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.052753396s ub 52.052753396s int 16s)) prio 255 cost 10 e36) pg 0x558e81938000 2019-12-07T19:34:13.605+0000 7fe2c5a62700 20 osd.0 op_wq _add_slot_waiter 1.4s1 peering, item epoch is 36, will wait on OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.052753396s ub 52.052753396s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.605+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:13.606+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 134 ==== pg_info2(1.1s1 1.1s1 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d800 con 0x558e818d7000 2019-12-07T19:34:13.606+0000 7fe2e7d3c700 15 osd.0 35 enqueue_peering_evt 1.1s1 epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:13.606+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.606+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 135 ==== pg_info2(1.fs1 1.fs1 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e8404d800 con 0x558e818d7000 2019-12-07T19:34:13.606+0000 7fe2e7d3c700 15 osd.0 35 enqueue_peering_evt 1.fs1 epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:13.606+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.606+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.606+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.606+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.606+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e36) pg 0x558e819e4000 2019-12-07T19:34:13.606+0000 7fe2cb26d700 20 osd.0 op_wq _add_slot_waiter 1.1s1 peering, item epoch is 36, will wait on OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.606+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:13.606+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.606+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.606+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.606+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e36) pg 0x558e81a9b000 2019-12-07T19:34:13.606+0000 7fe2c8267700 20 osd.0 op_wq _add_slot_waiter 1.fs1 peering, item epoch is 36, will wait on OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.606+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:13.609+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83ddbc00 0x558e80ef8b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:13.610+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e83ddbc00 0x558e80ef8b00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:13.611+0000 7fe2e753b700 10 osd.0 35 ms_handle_authentication new session 0x558e83ff7d40 con 0x558e83ddbc00 entity osd.1 addr 2019-12-07T19:34:13.611+0000 7fe2e753b700 10 osd.0 35 ms_handle_authentication session 0x558e83ff7d40 osd.1 has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:13.611+0000 7fe2e753b700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/37078 conn(0x558e83ddbc00 0x558e80ef8b00 crc :-1 s=READY pgs=3 cs=0 l=1 rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:13.611+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== osd.1 172.21.15.131:0/37078 1 ==== osd_op(osd.1.13:5 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e36) v8 ==== 226+0+29 (crc 0 0 0) 0x558e8448adc0 con 0x558e83ddbc00 2019-12-07T19:34:13.611+0000 7fe2e753b700 15 osd.0 35 enqueue_op 0x558e81943340 prio 63 cost 29 latency 0.000013 epoch 36 osd_op(osd.1.13:5 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e36) v8 2019-12-07T19:34:13.611+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(osd.1.13:5 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e36) v8) prio 63 cost 29 e36) 2019-12-07T19:34:13.611+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.611+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(osd.1.13:5 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e36) v8) prio 63 cost 29 e36) queued 2019-12-07T19:34:13.611+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.611+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(osd.1.13:5 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e36) v8) prio 63 cost 29 e36) pg 0x558e81075000 2019-12-07T19:34:13.611+0000 7fe2c6a64700 10 osd.0 35 dequeue_op 0x558e81943340 prio 63 cost 29 latency 0.000052 osd_op(osd.1.13:5 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e36) v8 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] 2019-12-07T19:34:13.611+0000 7fe2c6a64700 10 osd.0 35 maybe_share_map con 0x558e83ddbc00 172.21.15.131:0/37078 map epoch 0 -> 36 (as per caller) 2019-12-07T19:34:13.611+0000 7fe2c6a64700 20 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] do_request min 36, queue on waiting_for_map osd.1 2019-12-07T19:34:13.611+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> [v2:172.21.15.131:3301/0,v1:172.21.15.131:6790/0] -- mon_subscribe({osdmap=36}) v3 -- 0x558e84052c00 con 0x558e80f39000 2019-12-07T19:34:13.611+0000 7fe2c6a64700 10 osd.0 35 dequeue_op 0x558e81943340 finish 2019-12-07T19:34:13.611+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:13.611+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 262 ==== osd_map(36..36 src has 1..36) v4 ==== 677+0+0 (unknown 0 0 0) 0x558e8421efc0 con 0x558e818d7c00 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 osd.0 35 OSD::ms_dispatch: osd_map(36..36 src has 1..36) v4 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 osd.0 35 do_waiters -- start 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 osd.0 35 do_waiters -- finish 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 osd.0 35 _dispatch 0x558e8421efc0 osd_map(36..36 src has 1..36) v4 2019-12-07T19:34:13.611+0000 7fe2d4a80700 3 osd.0 35 handle_osd_map epochs [36,36], i have 35, src has [1,36] 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 osd.0 35 handle_osd_map got inc map for epoch 36 2019-12-07T19:34:13.611+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 263 ==== MOSDECSubOpWrite(1.8s5 36/28 ECSubWrite(tid=158, reqid=client.4540.0:284, at_version=36'33, trim_to=0'0, roll_forward_to=31'32)) v2 ==== 77981+0+0 (unknown 0 0 0) 0x558e845be800 con 0x558e818d7c00 2019-12-07T19:34:13.611+0000 7fe2e7d3c700 15 osd.0 35 enqueue_op 0x558e81943600 prio 127 cost 0 latency 0.000027 epoch 36 MOSDECSubOpWrite(1.8s5 36/28 ECSubWrite(tid=158, reqid=client.4540.0:284, at_version=36'33, trim_to=0'0, roll_forward_to=31'32)) v2 2019-12-07T19:34:13.611+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 36/28 ECSubWrite(tid=158, reqid=client.4540.0:284, at_version=36'33, trim_to=0'0, roll_forward_to=31'32)) v2) prio 127 cost 0 e36) 2019-12-07T19:34:13.611+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3138e826:::osdmap.35:0# 0x0~0 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1448 size 0x1448 (5192) 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e83f496c0 blob([0x40000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83f94620 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84549c70 blob([!~1000,0x41000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,448]) SharedBlob(0x558e84190230 sbid 0x0)) need 0x1000~448 cache has 0x[1000~448] 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3138e826:::osdmap.35:0# 0x0~1448 = 5192 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 osd.0 35 add_map_bl 35 5192 bytes 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 osd.0 35 got_full_map 36, nothing requested 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 osd.0 35 handle_osd_map pg_num_history pg_num_history(e36 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps purged_snaps {36={1=[4~1,6~2]}} 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore.onode(0x558e8125c500).flush flush done 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 1 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore.OmapIteratorImpl(0x558e8400d300) lower_bound to PSN__1_0000000000000003 key 0x00000000000000000000000000000403'.PSN__1_0000000000000003' 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore.OmapIteratorImpl(0x558e8400d300) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 snap_mapper._lookup_purged_snap pool 1 snap 3 key 'PSN__1_0000000000000003' lower_bound not found 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore.onode(0x558e8125c500).flush flush done 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 1 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore.OmapIteratorImpl(0x558e8400d300) lower_bound to PSN__1_0000000000000005 key 0x00000000000000000000000000000403'.PSN__1_0000000000000005' 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore.OmapIteratorImpl(0x558e8400d300) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 snap_mapper._lookup_purged_snap pool 1 snap 5 key 'PSN__1_0000000000000005' lower_bound not found 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore.onode(0x558e8125c500).flush flush done 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 1 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore.OmapIteratorImpl(0x558e8400d300) lower_bound to PSN__1_0000000000000005 key 0x00000000000000000000000000000403'.PSN__1_0000000000000005' 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore.OmapIteratorImpl(0x558e8400d300) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 snap_mapper._lookup_purged_snap pool 1 snap 5 key 'PSN__1_0000000000000005' lower_bound not found 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore.onode(0x558e8125c500).flush flush done 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) get_omap_iterator has_omap = 1 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore.OmapIteratorImpl(0x558e8400d300) lower_bound to PSN__1_0000000000000008 key 0x00000000000000000000000000000403'.PSN__1_0000000000000008' 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore.OmapIteratorImpl(0x558e8400d300) valid is at 0x00000000000000000000000000000403'~' 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 snap_mapper._lookup_purged_snap pool 1 snap 8 key 'PSN__1_0000000000000008' lower_bound not found 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps rm 0 keys, set 2 keys 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 osd.0 35 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e36 [1,36] lci=[13,36]) 2019-12-07T19:34:13.611+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e8441a300 seq 28 2019-12-07T19:34:13.611+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c2eb972f:::inc_osdmap.36:0# key 0x7f7fffffffffffffffc2eb97'/!inc_osdmap.36!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c2eb972f:::inc_osdmap.36:0# 0x0~279 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1164 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c2eb972f:::inc_osdmap.36:0# 0x0~279 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~279 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xd87 on front/back, now 0x0~1000 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8441a300 1 blobs 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x42000~1000/1000,1000,0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x42000~1000] 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6dea0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c310 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6dea0 blob([0x42000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c310 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e83e6dea0) get_ref 0x0~279 Blob(0x558e83e6dea0 blob([0x42000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c310 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e83e6dea0) get_ref init 0x1000, 1000 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~279: 0x0~279 Blob(0x558e83e6dea0 blob([0x42000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x279) SharedBlob(0x558e83e6c310 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e83e6c328 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:13.612+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== mon.2 v2:172.21.15.131:3301/0 29 ==== osd_map(36..36 src has 1..36) v4 ==== 677+0+0 (crc 0 0 0) 0x558e8421f200 con 0x558e80f39000 2019-12-07T19:34:13.612+0000 7fe2d959f700 20 osd.0 35 OSD::ms_dispatch: osd_map(36..36 src has 1..36) v4 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x279 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84640108) dirty_range mark inline shard dirty 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c2eb972f:::inc_osdmap.36:0# 0x0~279 = 0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:3a38e826:::osdmap.36:0# key 0x7f7fffffffffffffff3a38e8'&!osdmap.36!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3a38e826:::osdmap.36:0# 0x0~1454 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1165 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:3a38e826:::osdmap.36:0# 0x0~1454 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~454 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbac on front/back, now 0x0~1000 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8441a300 2 blobs 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x43000~2000/1000,2000,0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x43000~2000] 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84754e70 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84754e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84754e70 blob([0x43000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84754e00 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e84754e70) get_ref 0x0~1000 Blob(0x558e84754e70 blob([0x43000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84754e00 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e84754e70) get_ref init 0x1000, 1000 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e84754e70 blob([0x43000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e84754e00 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e84754e18 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84754d90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84754d20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84754d90 blob([!~1000,0x44000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84754d20 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e84754d90) get_ref 0x1000~454 Blob(0x558e84754d90 blob([!~1000,0x44000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e84754d20 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e84754d90) get_ref init 0x2000, 1000 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~454: 0x1000~454 Blob(0x558e84754d90 blob([!~1000,0x44000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558e84754d20 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e84754d38 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1454 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84640388) dirty_range mark inline shard dirty 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:3a38e826:::osdmap.36:0# 0x0~1454 = 0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8441a300 0 blobs 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_0000000000000004' <- PSN__1_0000000000000004 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000403'.PSN__1_0000000000000007' <- PSN__1_0000000000000007 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8441a300 0 blobs 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8441a300 cost 23428 (4 ios * 4000 + 7428 bytes) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8441a300 onodes 0x558e80251b80,0x558e817b9180,0x558e84640000,0x558e84640280 shared_blobs 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84640108) update #-1:c2eb972f:::inc_osdmap.36:0# 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84640108) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c2eb972f:::inc_osdmap.36:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84640388) update #-1:3a38e826:::osdmap.36:0# 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84640388) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:3a38e826:::osdmap.36:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8441a300 allocated 0x[42000~3000] released 0x[] 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 freelist allocate 0x42000~3000 2019-12-07T19:34:13.612+0000 7fe2d4a80700 20 freelist _xor first_key 0x0 last_key 0x0 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441a300 prepare 2019-12-07T19:34:13.612+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8441a300 2019-12-07T19:34:13.612+0000 7fe2d959f700 10 osd.0 35 do_waiters -- start 2019-12-07T19:34:13.612+0000 7fe2d959f700 10 osd.0 35 do_waiters -- finish 2019-12-07T19:34:13.612+0000 7fe2d959f700 20 osd.0 35 _dispatch 0x558e8421f200 osd_map(36..36 src has 1..36) v4 2019-12-07T19:34:13.612+0000 7fe2d959f700 3 osd.0 35 handle_osd_map epochs [36,36], i have 36, src has [1,36] 2019-12-07T19:34:13.612+0000 7fe2d959f700 10 osd.0 35 no new maps here, dropping 2019-12-07T19:34:13.612+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441a300 aio_wait 2019-12-07T19:34:13.612+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8441a300 2019-12-07T19:34:13.612+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441a300 io_done 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 1 stable 0 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:13.612+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8927 < min_flush_size 524288 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84640000 had 1 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84640280 had 1 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:13.612+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3859480576 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:13.612+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.99844) bluestore 86 GiB free (0.95994), bluefs_ratio 0.0399429 2019-12-07T19:34:13.612+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8998 < min_flush_size 524288 2019-12-07T19:34:13.612+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8998 < min_flush_size 524288 2019-12-07T19:34:13.612+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:13.612+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x14c7bf~2326 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:13.612+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x14c7bf 0x14c7bf~2326 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11c7bf 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x7bf 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xae5 and padding block with 0x51b 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x14eae5 2019-12-07T19:34:13.612+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.612+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:13.612+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 1 in 0.000146452s (0.000007506s flush + 0.000138946s kv commit) 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:13.612+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8441a300> 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <0x558e845eedc0> 2019-12-07T19:34:13.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441a300 kv_submitted 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8441a300 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e8441a300 osr 0x558e8107a500 2019-12-07T19:34:13.612+0000 7fe2d7285700 10 osd.0 35 _committed_osd_maps 36..36 2019-12-07T19:34:13.612+0000 7fe2d7285700 10 osd.0 35 advance to epoch 36 (<= last 36 <= newest_map 36) 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e84642000) prepare_write seq 26 0x1b000~1000 crc e6fdb5f9 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e84642000) prepare_write seq 26 0x2000~1000 crc 8181f5e2 2019-12-07T19:34:13.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72600 deferred_cleanup 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72600 onodes 0x558e80251b80,0x558e817b9180,0x558e8417c500,0x558e8417d900 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83e6c408 in 0x558e80219340) _finish_write discard buffer(0x558e846daf30 space 0x558e83e6c408 0x0~1000 writing nocache) 2019-12-07T19:34:13.612+0000 7fe2d7285700 7 osd.0 36 consume_map version 36 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e841900f8 in 0x558e80219340) _finish_write discard buffer(0x558e84759170 space 0x558e841900f8 0x0~1000 writing nocache) 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 36 identify_splits_and_merges 1.8s5 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84755668 in 0x558e80219340) _finish_write discard buffer(0x558e84759200 space 0x558e84755668 0x1000~1000 writing nocache) 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 36 identify_splits_and_merges 1.9s3 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 36 identify_splits_and_merges 1.1s1 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72600 done 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 36 identify_splits_and_merges 1.as3 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4db00 deferred_cleanup 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 36 identify_splits_and_merges 1.2s0 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72600 [] 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 36 identify_splits_and_merges 1.3s5 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.612+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 36 identify_splits_and_merges 1.bs1 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 36 identify_splits_and_merges 1.4s1 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 36 identify_splits_and_merges 1.5s4 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 36 identify_splits_and_merges 1.ds5 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 36 identify_splits_and_merges 1.es4 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 36 identify_splits_and_merges 1.7s3 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 36 identify_splits_and_merges 1.fs1 e35 to e36 pg_nums {18=16} 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 36 prune_pg_created 2019-12-07T19:34:13.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4db00 deferred_cleanup 2019-12-07T19:34:13.612+0000 7fe2d7285700 10 osd.0:0.consume_map 36 (was 35) 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4db00 onodes 0x558e80251b80,0x558e817b9180,0x558e83b09680,0x558e8402ef00 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 pending_peering first epoch 36 <= 36, requeueing 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e847552e8 in 0x558e80219340) _finish_write discard buffer(0x558e847594d0 space 0x558e847552e8 0x1000~1000 writing nocache) 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:0._wake_pg_slot _wake_pg_slot 1.8s5 to_process <> waiting <> waiting_peering {36=} 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e847553c8 in 0x558e80219340) _finish_write discard buffer(0x558e84759440 space 0x558e847553c8 0x0~1000 writing nocache) 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84755518 in 0x558e80219340) _finish_write discard buffer(0x558e847593b0 space 0x558e84755518 0x0~1000 writing nocache) 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4db00 done 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8441a000 deferred_cleanup 2019-12-07T19:34:13.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4db00 [] 2019-12-07T19:34:13.612+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.612+0000 7fe2d7285700 10 osd.0:1.consume_map 36 (was 35) 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:13.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441a000 deferred_cleanup 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 pending_peering first epoch 36 <= 36, requeueing 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8441a000 onodes 0x558e80251b80,0x558e817b9180,0x558e8402fb80,0x558e84314500 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83dd3128 in 0x558e80219340) _finish_write discard buffer(0x558e84759680 space 0x558e83dd3128 0x0~1000 writing nocache) 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.9s3 to_process <> waiting <> waiting_peering {36=} 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e847550b8 in 0x558e80219340) _finish_write discard buffer(0x558e847b0cf0 space 0x558e847550b8 0x1000~1000 writing nocache) 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84755198 in 0x558e80219340) _finish_write discard buffer(0x558e84759710 space 0x558e84755198 0x0~1000 writing nocache) 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8441a000 done 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 pending_peering first epoch 36 <= 36, requeueing 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1500 deferred_cleanup 2019-12-07T19:34:13.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8441a000 [] 2019-12-07T19:34:13.612+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:1._wake_pg_slot _wake_pg_slot 1.1s1 to_process <> waiting <> waiting_peering {36=} 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s)) prio 255 cost 10 e36) pg 0x558e841a4000 2019-12-07T19:34:13.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e819a1500 deferred_cleanup 2019-12-07T19:34:13.612+0000 7fe2d7285700 10 osd.0:2.consume_map 36 (was 35) 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e819a1500 onodes 0x558e80251b80,0x558e817b9180,0x558e83f29680,0x558e84770f00 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 pending_peering first epoch 36 <= 36, requeueing 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f94638 in 0x558e80219340) _finish_write discard buffer(0x558e847b13b0 space 0x558e83f94638 0x0~1000 writing nocache) 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:2._wake_pg_slot _wake_pg_slot 1.as3 to_process <> waiting <> waiting_peering {36=} 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83f95748 in 0x558e80219340) _finish_write discard buffer(0x558e840338c0 space 0x558e83f95748 0x0~1000 writing nocache) 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84190248 in 0x558e80219340) _finish_write discard buffer(0x558e847b1440 space 0x558e84190248 0x1000~1000 writing nocache) 2019-12-07T19:34:13.612+0000 7fe2d7285700 10 osd.0:3.consume_map 36 (was 35) 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e819a1500 done 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8441a300 deferred_queued 2019-12-07T19:34:13.612+0000 7fe2d7285700 10 osd.0:4.consume_map 36 (was 35) 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:13.612+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e819a1500 [] 2019-12-07T19:34:13.612+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 pending_peering first epoch 36 <= 36, requeueing 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:4._wake_pg_slot _wake_pg_slot 1.4s1 to_process <> waiting <> waiting_peering {36=} 2019-12-07T19:34:13.612+0000 7fe2d7285700 10 osd.0:5.consume_map 36 (was 35) 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:13.612+0000 7fe2d7285700 10 osd.0:6.consume_map 36 (was 35) 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 pending_peering first epoch 36 <= 36, requeueing 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:6._wake_pg_slot _wake_pg_slot 1.es4 to_process <> waiting <> waiting_peering {36=} 2019-12-07T19:34:13.612+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:13.612+0000 7fe2c7a66700 10 osd.0 pg_epoch: 35 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.612+0000 7fe2d7285700 10 osd.0:7.consume_map 36 (was 35) 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 osd.0:0.update_pg_epoch 1.8s5 35 -> 36 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 pending_peering first epoch 36 <= 36, requeueing 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s)) prio 255 cost 10 e36) pg 0x558e819e4000 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.7s3 to_process <> waiting <> waiting_peering {36=} 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 pending_peering first epoch 36 <= 36, requeueing 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0:7._wake_pg_slot _wake_pg_slot 1.fs1 to_process <> waiting <> waiting_peering {36=} 2019-12-07T19:34:13.612+0000 7fe2c7a66700 10 osd.0 pg_epoch: 35 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,4,5,2,3,0]/[1,4,5,2,3,0] -- 1/1 2019-12-07T19:34:13.612+0000 7fe2d7285700 10 osd.0 36 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 osd.0 pg_epoch: 35 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.612+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.612+0000 7fe2d7285700 15 osd.0 36 enqueue_peering_evt 1.8s5 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.1s1 35 -> 36 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7fe2d7285700 15 osd.0 36 enqueue_peering_evt 1.9s3 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 osd.0 pg_epoch: 35 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [3,0,2,4,5,1]/[3,0,2,4,5,1] -- 3/3 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.612+0000 7fe2d7285700 15 osd.0 36 enqueue_peering_evt 1.1s1 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7fe2d7285700 15 osd.0 36 enqueue_peering_evt 1.as3 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.612+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.8s5_head options 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7fe2d7285700 15 osd.0 36 enqueue_peering_evt 1.2s0 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.612+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.612+0000 7fe2d7285700 15 osd.0 36 enqueue_peering_evt 1.3s5 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.612+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 osd.0 pg_epoch: 36 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 osd.0 pg_epoch: 36 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.612+0000 7fe2d7285700 15 osd.0 36 enqueue_peering_evt 1.bs1 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 osd.0 pg_epoch: 36 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.612+0000 7fe2d7285700 15 osd.0 36 enqueue_peering_evt 1.4s1 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 osd.0 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 osd.0 pg_epoch: 36 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 osd.0 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.612+0000 7fe2d7285700 15 osd.0 36 enqueue_peering_evt 1.5s4 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.612+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 osd.0 pg_epoch: 36 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.612+0000 7fe2d7285700 15 osd.0 36 enqueue_peering_evt 1.ds5 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.612+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:13.612+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.612+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.612+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 osd.0 pg_epoch: 36 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.612+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558e83d07000 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.1s0( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115024122s ub 46.115024122s int 16s) 2019-12-07T19:34:13.612+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.8s0( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.113761808s ub 46.113761808s int 16s) 2019-12-07T19:34:13.612+0000 7fe2d7285700 15 osd.0 36 enqueue_peering_evt 1.es4 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.612+0000 7fe2d7285700 15 osd.0 36 enqueue_peering_evt 1.7s3 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.612+0000 7fe2d7285700 15 osd.0 36 enqueue_peering_evt 1.fs1 epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.612+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) 2019-12-07T19:34:13.612+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:13.612+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.5s4 35 -> 36 2019-12-07T19:34:13.612+0000 7fe2d7285700 7 osd.0 36 activate_map version 36 2019-12-07T19:34:13.612+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:13.612+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [2,1,3,4,0,5]/[2,1,3,4,0,5] -- 2/2 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b01e0 1.1s1_head 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107be00 = 0x558e83eb9200 seq 33 2019-12-07T19:34:13.612+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:13.612+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e84558c00 seq 41 2019-12-07T19:34:13.612+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.612+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# 2019-12-07T19:34:13.612+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.612+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._biginfo' <- _biginfo 2019-12-07T19:34:13.612+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s)) prio 255 cost 10 e36) pg 0x558e81a98000 2019-12-07T19:34:13.612+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:13.612+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._epoch' <- _epoch 2019-12-07T19:34:13.612+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.5s4_head options 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042c'._info' <- _info 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._biginfo' <- _biginfo 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.1s1_head 1#1:80000000::::head# = 0 2019-12-07T19:34:13.612+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83eb9200 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._epoch' <- _epoch 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83eb9200 onodes shared_blobs 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83eb9200 allocated 0x[] released 0x[] 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:13.612+0000 7fe2c9269700 20 osd.0 pg_epoch: 36 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.612+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:13.612+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84558c00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84558c00 onodes shared_blobs 2019-12-07T19:34:13.612+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84558c00 allocated 0x[] released 0x[] 2019-12-07T19:34:13.612+0000 7fe2c4a60700 10 osd.0 pg_epoch: 35 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.612+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb9200 prepare 2019-12-07T19:34:13.612+0000 7fe2c9269700 20 osd.0 pg_epoch: 36 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.612+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83eb9200 2019-12-07T19:34:13.612+0000 7fe2c9269700 20 osd.0 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb9200 io_done 2019-12-07T19:34:13.612+0000 7fe2c4a60700 20 osd.0:6.update_pg_epoch 1.es4 35 -> 36 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558c00 prepare 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84558c00 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558c00 io_done 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 35 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,3,4,5,0,2]/[1,3,4,5,0,2] -- 1/1 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s)) prio 255 cost 10 e36) pg 0x558e8192b000 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 36 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.613+0000 7fe2c9269700 20 osd.0 pg_epoch: 36 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558e841a4000 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 36 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 36 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.613+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 35 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.613+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558e819e7000 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0:1.update_pg_epoch 1.9s3 35 -> 36 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 36/28 ECSubWrite(tid=158, reqid=client.4540.0:284, at_version=36'33, trim_to=0'0, roll_forward_to=31'32)) v2) prio 127 cost 0 e36) queued 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 osd.0 pg_epoch: 36 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 36/28 ECSubWrite(tid=158, reqid=client.4540.0:284, at_version=36'33, trim_to=0'0, roll_forward_to=31'32)) v2) prio 127 cost 0 e36) pg 0x558e841a4000 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 35 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 osd.0 pg_epoch: 36 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 osd.0 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.613+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.ds5 35 -> 36 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 osd.0 36 dequeue_op 0x558e81943600 prio 127 cost 0 latency 0.001477 MOSDECSubOpWrite(1.8s5 36/28 ECSubWrite(tid=158, reqid=client.4540.0:284, at_version=36'33, trim_to=0'0, roll_forward_to=31'32)) v2 pg pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[a~1]] 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 36 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 osd.0 36 maybe_share_map con 0x558e818d7c00 v2:172.21.15.131:6823/37078 map epoch 35 -> 36 (as per caller) 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 35 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 36 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.613+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=5 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.8s5 36/28 ECSubWrite(tid=158, reqid=client.4540.0:284, at_version=36'33, trim_to=0'0, roll_forward_to=31'32)) v2 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 osd.0 pg_epoch: 36 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.613+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 pg_epoch: 36 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 36 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.es0( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114825888s ub 46.114825888s int 16s) 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 pg_epoch: 36 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 31'32 (0'0,31'32] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[a~1]] append_log log((0'0,31'32], crt=31'32) [36'33 (0'0) modify 1:15219530:::smithi13140796-21:head by client.4540.0:284 2019-12-07T19:34:13.610267+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 36 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.613+0000 7fe2c9269700 20 osd.0 pg_epoch: 36 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c9269700 20 osd.0 pg_epoch: 36 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.613+0000 7fe2c9269700 20 osd.0 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=31'32 crt=31'32 lcod 31'32 active mbc={} ps=[a~1]] add_log_entry 36'33 (0'0) modify 1:15219530:::smithi13140796-21:head by client.4540.0:284 2019-12-07T19:34:13.610267+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c4960 1.es4_head 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df7c0 = 0x558e83e72900 seq 29 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 pg_epoch: 36 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=31'32 crt=31'32 lcod 31'32 active mbc={} ps=[a~1]] append_log approx pg log length = 33 2019-12-07T19:34:13.613+0000 7fe2c4a60700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.9s0( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114035089s ub 46.114035089s int 16s) 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=31'32 crt=31'32 lcod 31'32 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._biginfo' <- _biginfo 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:13.613+0000 7fe2c9269700 20 osd.0 pg_epoch: 36 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._epoch' <- _epoch 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000430'._info' <- _info 2019-12-07T19:34:13.613+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 36'33, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.es4_head 4#1:70000000::::head# = 0 2019-12-07T19:34:13.613+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72900 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.613+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72900 onodes shared_blobs 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72900 allocated 0x[] released 0x[] 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b0b40 1.9s3_head 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de640 = 0x558e83eb9500 seq 40 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72900 prepare 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72900 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72900 io_done 2019-12-07T19:34:13.613+0000 7fe2c7265700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._biginfo' <- _biginfo 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._epoch' <- _epoch 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042d'._info' <- _info 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.9s3_head 3#1:90000000::::head# = 0 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83eb9500 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83eb9500 onodes shared_blobs 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83eb9500 allocated 0x[] released 0x[] 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558e81a98000 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e8417af00 seq 42 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb9500 prepare 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83eb9500 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb9500 io_done 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 osd.0 pg_epoch: 36 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.613+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1166 2019-12-07T19:34:13.613+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.8s5_head 5#1:15219530:::smithi13140796-21:head# = 0 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558e8192b000 2019-12-07T19:34:13.613+0000 7fe2cc26f700 20 osd.0 36 agent_entry empty queue 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 22 keys 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558e819e4000 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 osd.0 pg_epoch: 36 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.613+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s)) prio 255 cost 10 e36) pg 0x558e81a9b000 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 35 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.fs1 35 -> 36 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 35 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [3,0,5,4,1,2]/[3,0,5,4,1,2] -- 3/3 2019-12-07T19:34:13.613+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 134 ==== pg_info2(1.bs1 1.bs1 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83c46e00 con 0x558e818fe400 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.613+0000 7fe2e6d3a700 15 osd.0 36 enqueue_peering_evt 1.bs1 epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:13.613+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.613+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.8s5_head 5#1:15219530:::smithi13140796-21:head# 22 keys = 0 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.613+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.8s5_head 5#1:15219530:::smithi13140796-21:head# hinfo_key (42 bytes) 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 pg_epoch: 36 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.052753396s ub 52.052753396s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 pg_epoch: 36 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.052753396s ub 52.052753396s int 16s)) prio 255 cost 10 e36) pg 0x558e81938000 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:13.613+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 135 ==== pg_info2(1.3s5 1.3s5 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e83c46e00 con 0x558e818fe400 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.613+0000 7fe2e6d3a700 15 osd.0 36 enqueue_peering_evt 1.3s5 epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:13.613+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 pg_epoch: 36 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02000 had 1 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2236 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 3(0) info: 1.fs0( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.114604348s ub 46.114604348s int 16s) 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 osd.0:4.update_pg_epoch 1.4s1 35 -> 36 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3354 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 35 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02a00 had 1 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4472 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 36 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02280 had 1 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4495 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c000 1.fs1_head 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 4495 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966640 = 0x558e8417a300 seq 39 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 36 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x14eae5~118f to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:13.613+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x14eae5 0x14eae5~118f to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._biginfo' <- _biginfo 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~120000 x_off 0x11eae5 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xae5 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 36 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc74 and padding block with 0x38c 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._epoch' <- _epoch 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000433'._info' <- _info 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 36 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.fs1_head 1#1:f0000000::::head# = 0 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417a300 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417a300 onodes shared_blobs 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 36 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417a300 allocated 0x[] released 0x[] 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 osd.0 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 prepare 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417a300 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 36 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 io_done 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 36 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s)) prio 255 cost 10 e36) pg 0x558e8122e000 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 osd.0 pg_epoch: 36 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 36 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 7(0) info: 1.4s0( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.052753396s ub 52.052753396s int 16s) 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 36 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558e81231000 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 35 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0:7.update_pg_epoch 1.7s3 35 -> 36 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 35 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 35 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e83e73500 seq 15 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.3s5 35 -> 36 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.613+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 35 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x14fc74 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._biginfo' <- _biginfo 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 pg_epoch: 36 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 pg_epoch: 36 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:34:13.613+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.3s5_head options 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 4 cleaned 0 in 0.000157298s (0.000000086s flush + 0.000157212s kv commit) 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e73500 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e73500 onodes shared_blobs 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e73500 allocated 0x[] released 0x[] 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 908 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 pg_epoch: 36 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1129 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 pg_epoch: 36 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 pg_epoch: 36 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73500 prepare 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e73500 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03180 had 1 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73500 io_done 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1152 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.7s0( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.116162602s ub 46.116162602s int 16s) 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1152 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.8s5_head 5#1:15219530:::smithi13140796-21:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x14fc74~480 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558e81938000 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.0000000036.00000000000000000033' <- 0000000036.00000000000000000033 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x14fc74 0x14fc74~480 to file(ino 16 size 0x150000 mtime 2019-12-07T19:34:11.615155+0000 allocated 150000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~120000]) 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _allocate len 0xf4 from 0 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _allocate len 0xf4 from 1 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd280000 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd280000~10000/10000,10000,acd280000 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x160000 to include allocated 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8122c5a0 1.7s3_head 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 36 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 40 (was clean) 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966780 = 0x558e83e72000 seq 39 2019-12-07T19:34:13.613+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x160000 mtime 2019-12-07T19:34:13.614156+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~130000 x_off 0x11fc74 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc74 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417af00 cost 82421 (1 ios * 4000 + 78421 bytes) 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417af00 onodes 0x558e83cb2000 shared_blobs 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xf4 and padding block with 0xf0c 2019-12-07T19:34:13.613+0000 7fe2c425f700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e83cb2108) update 5#1:15219530:::smithi13140796-21:head# 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x160000 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._biginfo' <- _biginfo 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 bluestore.extentmap(0x558e83cb2108) update inline shard 2 bytes from 0 extents 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._epoch' <- _epoch 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000434'._info' <- _info 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.7s3_head 3#1:e0000000::::head# = 0 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e72000 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e72000 onodes shared_blobs 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e72000 allocated 0x[] released 0x[] 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 prepare 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e72000 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 io_done 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558e8122e000 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558e81a9b000 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:15219530:::smithi13140796-21:head# is 76576 (76568 bytes onode + 2 bytes spanning blobs + 6 bytes inline extents) 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 osd.0 pg_epoch: 36 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.613+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417af00 allocated 0x[] released 0x[] 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 prepare 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417af00 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 io_done 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s)) prio 255 cost 10 e36) pg 0x558e8193b000 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83eb9200,0x558e84558c00,0x558e83e72900,0x558e83eb9500> 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb9200 kv_submitted 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 lua=31'32 crt=31'32 lcod 31'32 active mbc={} ps=[a~1]] op_applied version 36'33 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83eb9200 2019-12-07T19:34:13.613+0000 7fe2c7a66700 10 osd.0 36 dequeue_op 0x558e81943600 finish 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb9200 finishing 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83eb9200 onodes 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83eb9200 done 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e8107be00 q now empty 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83eb9200 [] 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 35 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.as3 35 -> 36 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558c00 kv_submitted 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84558c00 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 pg_epoch: 36 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 35 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [1,4,2,0,5,3]/[1,4,2,0,5,3] -- 1/1 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558c00 finishing 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84558c00 onodes 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84558c00 done 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417af00 kv_queued 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558e83c78000 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84558c00 [] 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.613+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72900 kv_submitted 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72900 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72900 finishing 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72900 onodes 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72900 done 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df7c0 q now empty 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72900 [] 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 36 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb9500 kv_submitted 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83eb9500 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83eb9500 finishing 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83eb9500 onodes 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83eb9500 done 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 35 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de640 q now empty 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83eb9500 [] 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 36 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs1 35 -> 36 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 35 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:13.613+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x1500f4 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] state: Started advmap 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 36 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.613+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs1_head options 2019-12-07T19:34:13.613+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 1(0) info: 1.as0( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.115034256s ub 46.115034256s int 16s) 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] plpg_on_pool_change 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 pg_epoch: 36 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] hit_set_clear 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 pg_epoch: 36 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] agent_stop 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] handle_activate_map: 36 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c5e00 1.as3_head 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81966000 = 0x558e8195f500 seq 25 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] activate_map 2019-12-07T19:34:13.613+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._biginfo' <- _biginfo 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 pg_epoch: 36 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (40) on file(ino 16 size 0x160000 mtime 2019-12-07T19:34:13.614156+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]), flushing log 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._epoch' <- _epoch 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.613+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000432'._info' <- _info 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x160000 mtime 2019-12-07T19:34:13.614156+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 40 len 0x49 crc 0xbb1bdaf4) 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.as3_head 3#1:50000000::::head# = 0 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195f500 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x27000~1000 to file(ino 1 size 0x27000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195f500 onodes shared_blobs 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x27000 0x27000~1000 to file(ino 1 size 0x27000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195f500 allocated 0x[] released 0x[] 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x28000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x27000 2019-12-07T19:34:13.613+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 prepare 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195f500 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 io_done 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558e8193b000 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x28000 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 NullEvt) prio 255 cost 10 e36) pg 0x558e81075000 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000022 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] handle_advance_map: 36 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0:2.update_pg_epoch 1.2s0 35 -> 36 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 35 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[4~1,6~2,a~1]] state: Active advmap 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] on_active_advmap new purged_snaps [4~1,6~2], now [a~1] 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] share_pg_info 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 40 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x160000 mtime 2019-12-07T19:34:13.614156+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:13.613+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_info2(1.2s2 1.2s2 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 -- 0x558e84704700 con 0x558e818d7c00 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000398597s (0.000000082s flush + 0.000398515s kv commit) 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417a300> 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 4 submitting 4 deferred done 0 stable 0 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 kv_submitted 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417a300 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a300 finishing 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417a300 onodes 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a300 done 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966640 q now empty 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417a300 [] 2019-12-07T19:34:13.613+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.613+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:13.613+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:34:13.613+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_info2(1.2s5 1.2s5 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 -- 0x558e81a59500 con 0x558e818d7000 2019-12-07T19:34:13.613+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2236 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a03400 had 1 2019-12-07T19:34:13.613+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_info2(1.2s1 1.2s1 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 -- 0x558e81a58e00 con 0x558e818fe400 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 32524 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_info2(1.2s3 1.2s3 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 -- 0x558e81a5a300 con 0x558e818d6800 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 65292 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_info2(1.2s4 1.2s4 1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 -- 0x558e81a5aa00 con 0x558e818fe000 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 80255 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:13.613+0000 7fe2caa6c700 15 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] publish_stats_to_osd 34:107 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] state: Started advmap 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:13.613+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] hit_set_clear 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] agent_stop 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 36 advance_pg new pool opts old pool opts 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] handle_activate_map: 36 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] activate_map 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb2000 had 1 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] state: Active: handling ActMap 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 81373 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] Active: kicking snap trim 2019-12-07T19:34:13.613+0000 7fe2caa6c700 7 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] state: handle ActMap primary 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02f00 had 1 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 81396 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 81396 < min_flush_size 524288 2019-12-07T19:34:13.613+0000 7fe2caa6c700 15 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] publish_stats_to_osd 35:108 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x160000 mtime 2019-12-07T19:34:13.614156+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x1500f4~13df4 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:13.614156+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] activate_map: Not dirtying info: last_persisted is 34 while current is 36 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x1500f4 0x1500f4~13df4 to file(ino 16 size 0x160000 mtime 2019-12-07T19:34:13.614156+0000 allocated 160000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~130000]) 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _allocate len 0x3ee8 from 0 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _allocate len 0x3ee8 from 1 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate 0x10000/10000,10000,acd290000 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 fbmap_alloc 0x558e80233100 allocate extent: 0xacd290000~10000/10000,10000,acd290000 2019-12-07T19:34:13.613+0000 7fe2d7a86700 10 bluefs _flush_range extending WAL size to 0x170000 to include allocated 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range dirty_seq = 41 (was clean) 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] requeue_map_waiters osd.1 0x558e81943340 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~140000 x_off 0x1200f4 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xf4 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xee8 and padding block with 0x118 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 osd.0 op_wq(2) _enqueue_front OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(osd.1.13:5 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e36) v8) prio 63 cost 29 e36) 2019-12-07T19:34:13.613+0000 7fe2d7a86700 20 bluefs _flush_range zeroing WAL log up to 0x170000 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 NullEvt 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819b12c0 1.2s0_head 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de8c0 = 0x558e8195e600 seq 42 2019-12-07T19:34:13.613+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.613+0000 7fe2caa6c700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# 2019-12-07T19:34:13.613+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(osd.1.13:5 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e36) v8) prio 63 cost 29 e36) queued 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._biginfo' <- _biginfo 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._epoch' <- _epoch 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000429'._info' <- _info 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.2s0_head 0#1:40000000::::head# = 0 2019-12-07T19:34:13.613+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e600 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e600 onodes shared_blobs 2019-12-07T19:34:13.613+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e600 allocated 0x[] released 0x[] 2019-12-07T19:34:13.614+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 prepare 2019-12-07T19:34:13.614+0000 7fe2caa6c700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e600 2019-12-07T19:34:13.614+0000 7fe2caa6c700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 io_done 2019-12-07T19:34:13.614+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:13.614+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:13.613+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s)) prio 255 cost 10 e36) pg 0x558e83c78000 2019-12-07T19:34:13.614+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.614+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(osd.1.13:5 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e36) v8) prio 63 cost 29 e36) pg 0x558e81075000 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.bs0( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.104938842s ub 46.104938842s int 16s) 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.614+0000 7fe2c6a64700 10 osd.0 36 dequeue_op 0x558e81943340 prio 63 cost 29 latency 0.002729 osd_op(osd.1.13:5 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e36) v8 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e841981e0 1.bs1_head 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83de5540 = 0x558e83d4c300 seq 18 2019-12-07T19:34:13.614+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# 2019-12-07T19:34:13.614+0000 7fe2c6a64700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] _handle_message: osd_op(osd.1.13:5 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e36) v8 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._biginfo' <- _biginfo 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._epoch' <- _epoch 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000460'._info' <- _info 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.bs1_head 1#1:d0000000::::head# = 0 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83d4c300 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83d4c300 onodes shared_blobs 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83d4c300 allocated 0x[] released 0x[] 2019-12-07T19:34:13.614+0000 7fe2c6a64700 20 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op: op osd_op(osd.1.13:5 1.2s0 1:41df217a:::smithi13140796-39:head [copy-get max 8388608 in=29b] snapc 0=[] ondisk+read+known_if_redirected e36) v8 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 prepare 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83d4c300 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 io_done 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.614+0000 7fe2c6a64700 20 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] op_has_sufficient_caps session=0x558e83ff7d40 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s)) prio 255 cost 10 e36) pg 0x558e81231000 2019-12-07T19:34:13.614+0000 7fe2c6a64700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op osd_op(osd.1.13:5 1.2s0 1:41df217a:::smithi13140796-39:head [copy-get max 8388608 in=29b] snapc 0=[] ondisk+read+known_if_redirected e36) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 4(0) info: 1.3s0( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.105685108s ub 46.105685108s int 16s) 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.614+0000 7fe2c6a64700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] get_object_context: found obc in cache: 0x558e81222900 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e8417bb00 seq 18 2019-12-07T19:34:13.614+0000 7fe2c6a64700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] get_object_context: 0x558e81222900 1:41df217a:::smithi13140796-39:head rwstate(none n=0 w=0) oi: 1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) exists: 0 ssc: 0x558e83dcc6c0 snapset: 4=[]:{} 2019-12-07T19:34:13.614+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._biginfo' <- _biginfo 2019-12-07T19:34:13.614+0000 7fe2c6a64700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] find_object_context 1:41df217a:::smithi13140796-39:head @head oi=1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._epoch' <- _epoch 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417bb00 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.614+0000 7fe2c6a64700 25 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op oi 1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417bb00 onodes shared_blobs 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417bb00 allocated 0x[] released 0x[] 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417bb00 prepare 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417bb00 2019-12-07T19:34:13.614+0000 7fe2ca26b700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417bb00 io_done 2019-12-07T19:34:13.614+0000 7fe2c6a64700 20 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op obc obc(1:41df217a:::smithi13140796-39:head(dne) rwstate(read n=1 w=0)) 2019-12-07T19:34:13.614+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:13.614+0000 7fe2c6a64700 20 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] fill_in_copy_get_noent got reqids [client.4540.0:249,0,client.4540.0:231,12,client.4540.0:116,5,client.4540.0:115,1] 2019-12-07T19:34:13.614+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/37078 -- osd_op_reply(5 smithi13140796-39 [copy-get max 8388608 out=255b] v0'0 uv0 ondisk = -2 ((2) No such file or directory)) v8 -- 0x558e846c3180 con 0x558e83ddbc00 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x163ee8 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:13.614+0000 7fe2c6a64700 10 osd.0 36 dequeue_op 0x558e81943340 finish 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:13.614+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluefs _fsync file metadata was dirty (41) on file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]), flushing log 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log 1 dirty_files 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log op_file_update file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs _flush_and_sync_log txn(seq 41 len 0x49 crc 0xac182917) 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e803a5a40 0x28000~1000 to file(ino 1 size 0x28000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e803a5a40 pos 0x28000 0x28000~1000 to file(ino 1 size 0x28000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 1 size 0x29000 mtime 0.000000 allocated 400000 extents [1:0xacccc0000~400000]) 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacccc0000~400000 x_off 0x28000 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e803a5a40 pos now 0x29000 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 1 aios 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e803a5a40 done in 0.000186 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log log_seq_stable 41 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluefs _flush_and_sync_log cleaned file file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 4 cleaned 0 in 0.001127245s (0.000000083s flush + 0.001127162s kv commit) 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 3 submitting 3 deferred done 0 stable 0 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e819b6a00 had 1 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2236 < min_flush_size 524288 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83c72780 had 1 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3354 < min_flush_size 524288 2019-12-07T19:34:13.614+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3377 < min_flush_size 524288 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 3377 < min_flush_size 524288 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x163ee8~d31 to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.614+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x163ee8 0x163ee8~d31 to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.615+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.615+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~140000 x_off 0x133ee8 2019-12-07T19:34:13.615+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xee8 2019-12-07T19:34:13.615+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.615+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc19 and padding block with 0x3e7 2019-12-07T19:34:13.615+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x164c19 2019-12-07T19:34:13.615+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.615+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:13.615+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:13.615+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:13.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 3 cleaned 0 in 0.000100859s (0.000000052s flush + 0.000100807s kv commit) 2019-12-07T19:34:13.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:13.615+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 52 ==== pg_info2(1.ds5 1.ds5 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e81a5a300 con 0x558e818d6000 2019-12-07T19:34:13.615+0000 7fe2e753b700 15 osd.0 36 enqueue_peering_evt 1.ds5 epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.090950317s ub 52.090950317s int 16s) 2019-12-07T19:34:13.615+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.090950317s ub 52.090950317s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.615+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.615+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.090950317s ub 52.090950317s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.615+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.615+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.090950317s ub 52.090950317s int 16s)) prio 255 cost 10 e36) pg 0x558e819e7000 2019-12-07T19:34:13.615+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:13.615+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.ds0( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) pg_lease(ru 52.090950317s ub 52.090950317s int 16s) 2019-12-07T19:34:13.615+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.615+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:13.615+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e817b5860 1.ds5_head 2019-12-07T19:34:13.615+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e817952c0 = 0x558e845d5200 seq 22 2019-12-07T19:34:13.615+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# 2019-12-07T19:34:13.615+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._biginfo' <- _biginfo 2019-12-07T19:34:13.615+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._epoch' <- _epoch 2019-12-07T19:34:13.615+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000042f'._info' <- _info 2019-12-07T19:34:13.615+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.ds5_head 5#1:b0000000::::head# = 0 2019-12-07T19:34:13.615+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e845d5200 cost 5275 (1 ios * 4000 + 1275 bytes) 2019-12-07T19:34:13.615+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e845d5200 onodes shared_blobs 2019-12-07T19:34:13.615+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e845d5200 allocated 0x[] released 0x[] 2019-12-07T19:34:13.615+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d5200 prepare 2019-12-07T19:34:13.615+0000 7fe2c9269700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e845d5200 2019-12-07T19:34:13.615+0000 7fe2c9269700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d5200 io_done 2019-12-07T19:34:13.615+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:13.615+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 53 ==== pg_info2(1.5s4 1.5s4 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) e36/36) v1 ==== 1060+0+0 (unknown 0 0 0) 0x558e81a5a300 con 0x558e818d6000 2019-12-07T19:34:13.615+0000 7fe2e753b700 15 osd.0 36 enqueue_peering_evt 1.5s4 epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:13.615+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e36) 2019-12-07T19:34:13.615+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.615+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e36) queued 2019-12-07T19:34:13.615+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.615+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s)) prio 255 cost 10 e36) pg 0x558e83d07000 2019-12-07T19:34:13.615+0000 7fe2c5261700 10 osd.0 pg_epoch: 36 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] do_peering_event: epoch_sent: 36 epoch_requested: 36 MInfoRec from 2(0) info: 1.5s0( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) pg_lease(ru 46.131453992s ub 46.131453992s int 16s) 2019-12-07T19:34:13.615+0000 7fe2c5261700 10 osd.0 pg_epoch: 36 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[4~1,6~2,a~1]] proc_primary_info updating purged_snaps to [a~1] 2019-12-07T19:34:13.615+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:13.615+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403fe00 1.5s4_head 2019-12-07T19:34:13.615+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811de3c0 = 0x558e84558000 seq 20 2019-12-07T19:34:13.615+0000 7fe2c5261700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# 2019-12-07T19:34:13.615+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._biginfo' <- _biginfo 2019-12-07T19:34:13.615+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._epoch' <- _epoch 2019-12-07T19:34:13.615+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000462'._info' <- _info 2019-12-07T19:34:13.615+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.5s4_head 4#1:a0000000::::head# = 0 2019-12-07T19:34:13.615+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e84558000 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:13.615+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e84558000 onodes shared_blobs 2019-12-07T19:34:13.615+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e84558000 allocated 0x[] released 0x[] 2019-12-07T19:34:13.615+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558000 prepare 2019-12-07T19:34:13.615+0000 7fe2c5261700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e84558000 2019-12-07T19:34:13.615+0000 7fe2c5261700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558000 io_done 2019-12-07T19:34:13.615+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:13.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:13.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 2 submitting 2 deferred done 0 stable 0 2019-12-07T19:34:13.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1118 < min_flush_size 524288 2019-12-07T19:34:13.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02780 had 1 2019-12-07T19:34:13.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2406 < min_flush_size 524288 2019-12-07T19:34:13.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83cb3400 had 1 2019-12-07T19:34:13.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2429 < min_flush_size 524288 2019-12-07T19:34:13.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 2429 < min_flush_size 524288 2019-12-07T19:34:13.615+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x164c19~97d to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.615+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x164c19 0x164c19~97d to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.615+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.616+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~140000 x_off 0x134c19 2019-12-07T19:34:13.616+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc19 2019-12-07T19:34:13.616+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.616+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x596 and padding block with 0xa6a 2019-12-07T19:34:13.616+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x165596 2019-12-07T19:34:13.616+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.616+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:13.616+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:13.616+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:13.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 2 cleaned 0 in 0.000179709s (0.000000061s flush + 0.000179648s kv commit) 2019-12-07T19:34:13.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e73500,0x558e83e72000,0x558e8417af00,0x558e8195f500,0x558e8195e600,0x558e83d4c300,0x558e8417bb00,0x558e845d5200,0x558e84558000> 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73500 kv_submitted 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e73500 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73500 finishing 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e73500 onodes 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e73500 done 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df900 q now empty 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e73500 [] 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.616+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 kv_submitted 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e72000 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e72000 finishing 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e72000 onodes 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e72000 done 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966780 q now empty 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e72000 [] 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.616+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 kv_submitted 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417af00 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417af00 finishing 2019-12-07T19:34:13.616+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417af00 onodes 0x558e83cb2000 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417af00 done 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417af00 [] 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 kv_submitted 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195f500 2019-12-07T19:34:13.616+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195f500 finishing 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195f500 onodes 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195f500 done 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81966000 q now empty 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195f500 [] 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 kv_submitted 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e600 2019-12-07T19:34:13.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:13.616+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 36/28 ECSubWriteReply(tid=158, last_complete=36'33, committed=1, applied=1)) v2 -- 0x558e810fc200 con 0x558e818d7c00 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e600 finishing 2019-12-07T19:34:13.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e600 onodes 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e600 done 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de8c0 q now empty 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e600 [] 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 kv_submitted 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83d4c300 2019-12-07T19:34:13.616+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83d4c300 finishing 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83d4c300 onodes 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83d4c300 done 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83de5540 q now empty 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83d4c300 [] 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417bb00 kv_submitted 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417bb00 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417bb00 finishing 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417bb00 onodes 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417bb00 done 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417bb00 [] 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d5200 kv_submitted 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e845d5200 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e845d5200 finishing 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e845d5200 onodes 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e845d5200 done 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e817952c0 q now empty 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e845d5200 [] 2019-12-07T19:34:13.616+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558000 kv_submitted 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e84558000 2019-12-07T19:34:13.616+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e84558000 finishing 2019-12-07T19:34:13.616+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e84558000 onodes 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e84558000 done 2019-12-07T19:34:13.616+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811de3c0 q now empty 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e84558000 [] 2019-12-07T19:34:13.616+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:13.616+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:13.616+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:13.616+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:13.617+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 264 ==== MOSDECSubOpWrite(1.8s5 36/28 ECSubWrite(tid=159, reqid=client.4540.0:284, at_version=0'0, trim_to=0'0, roll_forward_to=36'33)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e845bf000 con 0x558e818d7c00 2019-12-07T19:34:13.617+0000 7fe2e7d3c700 15 osd.0 36 enqueue_op 0x558e81943a20 prio 127 cost 0 latency 0.000023 epoch 36 MOSDECSubOpWrite(1.8s5 36/28 ECSubWrite(tid=159, reqid=client.4540.0:284, at_version=0'0, trim_to=0'0, roll_forward_to=36'33)) v2 2019-12-07T19:34:13.617+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 36/28 ECSubWrite(tid=159, reqid=client.4540.0:284, at_version=0'0, trim_to=0'0, roll_forward_to=36'33)) v2) prio 127 cost 0 e36) 2019-12-07T19:34:13.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:13.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 36/28 ECSubWrite(tid=159, reqid=client.4540.0:284, at_version=0'0, trim_to=0'0, roll_forward_to=36'33)) v2) prio 127 cost 0 e36) queued 2019-12-07T19:34:13.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:13.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGOpItem(op=MOSDECSubOpWrite(1.8s5 36/28 ECSubWrite(tid=159, reqid=client.4540.0:284, at_version=0'0, trim_to=0'0, roll_forward_to=36'33)) v2) prio 127 cost 0 e36) pg 0x558e841a4000 2019-12-07T19:34:13.617+0000 7fe2c7a66700 10 osd.0 36 dequeue_op 0x558e81943a20 prio 127 cost 0 latency 0.000048 MOSDECSubOpWrite(1.8s5 36/28 ECSubWrite(tid=159, reqid=client.4540.0:284, at_version=0'0, trim_to=0'0, roll_forward_to=36'33)) v2 pg pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[a~1]] 2019-12-07T19:34:13.617+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.8s5 36/28 ECSubWrite(tid=159, reqid=client.4540.0:284, at_version=0'0, trim_to=0'0, roll_forward_to=36'33)) v2 2019-12-07T19:34:13.617+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=31'32 active mbc={} ps=[a~1]] append_log log((0'0,36'33], crt=31'32) [] 2019-12-07T19:34:13.617+0000 7fe2c7a66700 20 osd.0 pg_epoch: 36 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] rollforward: entry=36'33 (0'0) modify 1:15219530:::smithi13140796-21:head by client.4540.0:284 2019-12-07T19:34:13.610267+0000 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:13.617+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] append_log approx pg log length = 33 2019-12-07T19:34:13.617+0000 7fe2c7a66700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:13.617+0000 7fe2c7a66700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:13.617+0000 7fe2c7a66700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:13.617+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403ed20 1.8s5_head 2019-12-07T19:34:13.617+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e81967400 = 0x558e8417a600 seq 43 2019-12-07T19:34:13.617+0000 7fe2c7a66700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# 2019-12-07T19:34:13.617+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'._info' <- _info 2019-12-07T19:34:13.617+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:13.617+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x000000000000045f'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:13.617+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.8s5_head 5#1:10000000::::head# = 0 2019-12-07T19:34:13.617+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8417a600 cost 5303 (1 ios * 4000 + 1303 bytes) 2019-12-07T19:34:13.617+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8417a600 onodes shared_blobs 2019-12-07T19:34:13.617+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8417a600 allocated 0x[] released 0x[] 2019-12-07T19:34:13.617+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 prepare 2019-12-07T19:34:13.617+0000 7fe2c7a66700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8417a600 2019-12-07T19:34:13.617+0000 7fe2c7a66700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 io_done 2019-12-07T19:34:13.617+0000 7fe2c7a66700 10 osd.0 36 dequeue_op 0x558e81943a20 finish 2019-12-07T19:34:13.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:13.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:13.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:13.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1114 < min_flush_size 524288 2019-12-07T19:34:13.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8417c780 had 1 2019-12-07T19:34:13.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:13.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1137 < min_flush_size 524288 2019-12-07T19:34:13.617+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.617+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x165596~471 to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.617+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x165596 0x165596~471 to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.617+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:13.617+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~140000 x_off 0x135596 2019-12-07T19:34:13.617+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x596 2019-12-07T19:34:13.617+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:13.617+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xa07 and padding block with 0x5f9 2019-12-07T19:34:13.617+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x165a07 2019-12-07T19:34:13.617+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:13.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:13.617+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:13.617+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:13.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000153004s (0.000000055s flush + 0.000152949s kv commit) 2019-12-07T19:34:13.617+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:13.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:13.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8417a600> 2019-12-07T19:34:13.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:13.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 kv_submitted 2019-12-07T19:34:13.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8417a600 2019-12-07T19:34:13.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8417a600 finishing 2019-12-07T19:34:13.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8417a600 onodes 2019-12-07T19:34:13.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8417a600 done 2019-12-07T19:34:13.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e81967400 q now empty 2019-12-07T19:34:13.617+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8417a600 [] 2019-12-07T19:34:13.617+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:13.617+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:13.617+0000 7fe2cba6e700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- MOSDECSubOpWriteReply(1.8s0 36/28 ECSubWriteReply(tid=159, last_complete=36'33, committed=1, applied=1)) v2 -- 0x558e83fee000 con 0x558e818d7c00 2019-12-07T19:34:13.617+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:13.617+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:13.632+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.682+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.732+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.783+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.833+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.883+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.933+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:13.983+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.033+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.083+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.133+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.183+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.233+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.284+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.334+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.384+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.425+0000 7fe2e2ca2700 10 osd.0 36 tick_without_osd_lock 2019-12-07T19:34:14.425+0000 7fe2e2ca2700 20 bluestore(/var/lib/ceph/osd/ceph-0) statfs store_statfs(0x163fa7d000/0x40000000/0x1680000000, data 0x5466ed/0x581000, compress 0x0/0x0/0x0, omap 0x0, meta 0x40000000) 2019-12-07T19:34:14.425+0000 7fe2e2ca2700 25 osd.0 36 heartbeat_check osd.1 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955408+0000 last_rx_front 2019-12-07T19:34:10.955347+0000 2019-12-07T19:34:14.425+0000 7fe2e2ca2700 25 osd.0 36 heartbeat_check osd.2 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955302+0000 last_rx_front 2019-12-07T19:34:10.955288+0000 2019-12-07T19:34:14.425+0000 7fe2e2ca2700 25 osd.0 36 heartbeat_check osd.3 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955335+0000 last_rx_front 2019-12-07T19:34:10.955354+0000 2019-12-07T19:34:14.425+0000 7fe2e2ca2700 25 osd.0 36 heartbeat_check osd.4 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955428+0000 last_rx_front 2019-12-07T19:34:10.955392+0000 2019-12-07T19:34:14.425+0000 7fe2e2ca2700 25 osd.0 36 heartbeat_check osd.5 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955481+0000 last_rx_front 2019-12-07T19:34:10.955444+0000 2019-12-07T19:34:14.425+0000 7fe2e2ca2700 25 osd.0 36 heartbeat_check osd.7 first_tx 2019-12-07T19:33:41.252061+0000 last_tx 2019-12-07T19:34:10.955047+0000 last_rx_back 2019-12-07T19:34:10.955373+0000 last_rx_front 2019-12-07T19:34:10.955455+0000 2019-12-07T19:34:14.425+0000 7fe2e2ca2700 20 osd.0 36 scrub_random_backoff lost coin flip, randomly backing off 2019-12-07T19:34:14.425+0000 7fe2e2ca2700 10 osd.0 36 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2019-12-07T19:34:14.425+0000 7fe2e2ca2700 20 osd.0 36 promote_throttle_recalibrate new_prob 1000 2019-12-07T19:34:14.425+0000 7fe2e2ca2700 10 osd.0 36 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2019-12-07T19:34:14.434+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.484+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.534+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.538+0000 7fe2e4535700 10 osd.0 36 tick 2019-12-07T19:34:14.538+0000 7fe2e4535700 10 osd.0 36 do_waiters -- start 2019-12-07T19:34:14.538+0000 7fe2e4535700 10 osd.0 36 do_waiters -- finish 2019-12-07T19:34:14.538+0000 7fe2e4535700 20 osd.0 36 tick last_purged_snaps_scrub 2019-12-07T19:33:37.558614+0000 next 2019-12-07T19:34:37.558614+0000 2019-12-07T19:34:14.554+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.1 172.21.15.131:0/37078 6 ==== osd_ping(ping e36 up_from 13 ping_stamp 2019-12-07T19:34:14.554988+0000/38.011911392s send_stamp 38.011911392s delta_ub 0.002416414s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e818a9400 con 0x558e81984c00 2019-12-07T19:34:14.554+0000 7fe2e6d3a700 20 osd.0 36 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002083743s,0.002416414s]) 2019-12-07T19:34:14.554+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e36 up_from 13 ping_stamp 2019-12-07T19:34:14.554988+0000/38.011911392s send_stamp 38.009827649s delta_ub 18446744073.707467873s) v5 -- 0x558e845c2e00 con 0x558e81984c00 2019-12-07T19:34:14.554+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.1 172.21.15.131:0/37078 6 ==== osd_ping(ping e36 up_from 13 ping_stamp 2019-12-07T19:34:14.554988+0000/38.011911392s send_stamp 38.011911392s delta_ub 0.002416414s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83fee000 con 0x558e81985000 2019-12-07T19:34:14.554+0000 7fe2e7d3c700 20 osd.0 36 handle_osd_ping new stamps hbstamp(osd.1 up_from 13 peer_clock_delta [0.002061166s,0.002416414s]) 2019-12-07T19:34:14.554+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.131:0/37078 -- osd_ping(ping_reply e36 up_from 13 ping_stamp 2019-12-07T19:34:14.554988+0000/38.011911392s send_stamp 38.009850226s delta_ub 18446744073.707490450s) v5 -- 0x558e810fc200 con 0x558e81985000 2019-12-07T19:34:14.584+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 689809 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.614+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 137 ==== pg_update_log_missing(1.3s5 epoch 37/28 rep_tid 59 entries 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 ==== 258+0+0 (unknown 0 0 0) 0x558e8419c240 con 0x558e818fe400 2019-12-07T19:34:14.614+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 136 ==== osd_map(35..37 src has 1..37) v4 ==== 1871+0+0 (unknown 0 0 0) 0x558e8419cb40 con 0x558e818fe400 2019-12-07T19:34:14.614+0000 7fe2d4a80700 20 osd.0 36 OSD::ms_dispatch: osd_map(35..37 src has 1..37) v4 2019-12-07T19:34:14.614+0000 7fe2d4a80700 10 osd.0 36 do_waiters -- start 2019-12-07T19:34:14.614+0000 7fe2e6d3a700 15 osd.0 36 enqueue_op 0x558e83dd89a0 prio 127 cost 0 latency 0.000029 epoch 37 pg_update_log_missing(1.3s5 epoch 37/28 rep_tid 59 entries 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 2019-12-07T19:34:14.614+0000 7fe2d4a80700 10 osd.0 36 do_waiters -- finish 2019-12-07T19:34:14.614+0000 7fe2d4a80700 20 osd.0 36 _dispatch 0x558e8419cb40 osd_map(35..37 src has 1..37) v4 2019-12-07T19:34:14.614+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=pg_update_log_missing(1.3s5 epoch 37/28 rep_tid 59 entries 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e37) 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=pg_update_log_missing(1.3s5 epoch 37/28 rep_tid 59 entries 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.615+0000 7fe2d4a80700 3 osd.0 36 handle_osd_map epochs [35,37], i have 36, src has [1,37] 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 osd.0 36 handle_osd_map got inc map for epoch 37 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=pg_update_log_missing(1.3s5 epoch 37/28 rep_tid 59 entries 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3) prio 127 cost 0 e37) pg 0x558e81231000 2019-12-07T19:34:14.615+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3a38e826:::osdmap.36:0# 0x0~0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read 0x0~1454 size 0x1454 (5204) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_read will do buffered read 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84754e70 blob([0x43000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e84754e00 sbid 0x0)) need 0x0~1000 cache has 0x[0~1000] 2019-12-07T19:34:14.615+0000 7fe2c6263700 10 osd.0 36 dequeue_op 0x558e83dd89a0 prio 127 cost 0 latency 0.000095 pg_update_log_missing(1.3s5 epoch 37/28 rep_tid 59 entries 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 pg pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _read_cache blob Blob(0x558e84754d90 blob([!~1000,0x44000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,454]) SharedBlob(0x558e84754d20 sbid 0x0)) need 0x1000~454 cache has 0x[1000~454] 2019-12-07T19:34:14.615+0000 7fe2c6263700 10 osd.0 36 maybe_share_map con 0x558e818fe400 v2:172.21.15.109:6810/36938 map epoch 34 -> 37 (as per caller) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) read meta #-1:3a38e826:::osdmap.36:0# 0x0~1454 = 5204 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 osd.0 36 add_map_bl 36 5204 bytes 2019-12-07T19:34:14.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 36 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] _handle_message: pg_update_log_missing(1.3s5 epoch 37/28 rep_tid 59 entries 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 trim_to 0'0 roll_forward_to 0'0) v3 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 osd.0 pg_epoch: 36 pg[1.3s5( v 23'10 (0'0,23'10] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] do_update_log_missing op_trim_to = -- op_roll_forward_to = -- 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 update missing, append 37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 osd.0 pg_epoch: 36 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 lua=23'10 crt=23'10 lcod 23'10 active mbc={} ps=[a~1]] append_log_entries_update_missing trim_to bool = 0 trim_to = 0'0 2019-12-07T19:34:14.615+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 37'11, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.615+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e8195fb00 seq 19 2019-12-07T19:34:14.615+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.0000000037.00000000000000000011' <- 0000000037.00000000000000000011 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._fastinfo' <- _fastinfo 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 osd.0 36 got_full_map 37, nothing requested 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.615+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:14.615+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195fb00 cost 4717 (1 ios * 4000 + 717 bytes) 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195fb00 onodes shared_blobs 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195fb00 allocated 0x[] released 0x[] 2019-12-07T19:34:14.615+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 prepare 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195fb00 2019-12-07T19:34:14.615+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 io_done 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 osd.0 36 handle_osd_map pg_num_history pg_num_history(e37 pg_nums {1={18=16}} deleted_pools ) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps purged_snaps {37={}} 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 snap_mapper.record_purged_snaps rm 0 keys, set 0 keys 2019-12-07T19:34:14.615+0000 7fe2c6263700 10 osd.0 pg_epoch: 36 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 lua=23'10 crt=23'10 lcod 23'10 active mbc={} ps=[a~1]] op_applied version 37'11 2019-12-07T19:34:14.615+0000 7fe2c6263700 10 osd.0 36 dequeue_op 0x558e83dd89a0 finish 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 osd.0 36 write_superblock sb(ad52b421-bda1-4750-8dc4-c9e4d1297c01 osd.0 b39c74bd-b656-4393-a258-c2eebf51a555 e37 [1,37] lci=[13,37]) 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e80230960 meta 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e8107a500 = 0x558e8441a600 seq 29 2019-12-07T19:34:14.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:14.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:c82b972f:::inc_osdmap.37:0# key 0x7f7fffffffffffffffc82b97'/!inc_osdmap.37!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:14.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 567 < min_flush_size 524288 2019-12-07T19:34:14.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c82b972f:::inc_osdmap.37:0# 0x0~24d 2019-12-07T19:34:14.615+0000 7fe2d7a86700 10 bluefs get_usage bdev 1 free 3859349504 (3.6 GiB) / 3865509888 (3.6 GiB), used 0% 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1167 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:c82b972f:::inc_osdmap.37:0# 0x0~24d - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24d 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xdb3 on front/back, now 0x0~1000 2019-12-07T19:34:14.615+0000 7fe2d7a86700 10 bluestore(/var/lib/ceph/osd/ceph-0) _get_bluefs_size_delta bluefs 3.6 GiB free (0.998406) bluestore 86 GiB free (0.95994), bluefs_ratio 0.0399416 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8441a600 1 blobs 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x1000/1000,1000,0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x4bc000~1000/1000,1000,0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x4bc000~1000] 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83e6c2a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84014ee0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:14.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 590 < min_flush_size 524288 2019-12-07T19:34:14.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 590 < min_flush_size 524288 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83e6c2a0 blob([0x4bc000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84014ee0 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e83e6c2a0) get_ref 0x0~24d Blob(0x558e83e6c2a0 blob([0x4bc000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84014ee0 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e83e6c2a0) get_ref init 0x1000, 1000 2019-12-07T19:34:14.615+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~24d: 0x0~24d Blob(0x558e83e6c2a0 blob([0x4bc000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24d) SharedBlob(0x558e84014ee0 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x165a07~24e to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e84014ef8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:14.615+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x165a07 0x165a07~24e to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x24d 2019-12-07T19:34:14.615+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.615+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~140000 x_off 0x135a07 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84640608) dirty_range mark inline shard dirty 2019-12-07T19:34:14.615+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xa07 2019-12-07T19:34:14.615+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:c82b972f:::inc_osdmap.37:0# 0x0~24d = 0 2019-12-07T19:34:14.615+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xc55 and padding block with 0x3ab 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) get_onode oid #-1:37b8e826:::osdmap.37:0# key 0x7f7fffffffffffffff37b8e8'&!osdmap.37!='0x0000000000000000ffffffffffffffff'o' 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(meta 0x558e80230960) r -2 v.len 0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:37b8e826:::osdmap.37:0# 0x0~1428 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1168 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:37b8e826:::osdmap.37:0# 0x0~1428 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~1000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x1000~428 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _pad_zeros pad 0x0 + 0xbd8 on front/back, now 0x0~1000 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8441a600 2 blobs 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate 0x2000/1000,2000,0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x4bd000~2000/1000,2000,0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x4bd000~2000] 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7b7a0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194ee70 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7b7a0 blob([0x4bd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194ee70 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e81a7b7a0) get_ref 0x0~1000 Blob(0x558e81a7b7a0 blob([0x4bd000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194ee70 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e81a7b7a0) get_ref init 0x1000, 1000 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~1000: 0x0~1000 Blob(0x558e81a7b7a0 blob([0x4bd000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8194ee70 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e8194ee88 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:14.615+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x165c55 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write forcing blob_offset to 0x1000 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194e700 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c000 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.615+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.615+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194e700 blob([!~1000,0x4be000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c000 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:14.615+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e8194e700) get_ref 0x1000~428 Blob(0x558e8194e700 blob([!~1000,0x4be000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x0 0x0) SharedBlob(0x558e83e6c000 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e8194e700) get_ref init 0x2000, 1000 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1000~428: 0x1000~428 Blob(0x558e8194e700 blob([!~1000,0x4be000~1000] csum+has_unused crc32c/0x1000 unused=0xff) use_tracker(0x2*0x1000 0x[0,428]) SharedBlob(0x558e83e6c000 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e83e6c018 in 0x558e80219340) _discard 0x1000~1000 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x1428 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84640888) dirty_range mark inline shard dirty 2019-12-07T19:34:14.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000153330s (0.000000204s flush + 0.000153126s kv commit) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:37b8e826:::osdmap.37:0# 0x0~1428 = 0 2019-12-07T19:34:14.615+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:14.615+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 2019-12-07T19:34:14.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:46e858f0:::pg_num_history:head# 0x0~26 - have 0x26 (38) bytes fadvise_flags 0x0 2019-12-07T19:34:14.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195fb00> 2019-12-07T19:34:14.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~26 2019-12-07T19:34:14.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 kv_submitted 2019-12-07T19:34:14.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195fb00 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) bstart 0x0 2019-12-07T19:34:14.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195fb00 finishing 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xfda 2019-12-07T19:34:14.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195fb00 onodes 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:14.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195fb00 done 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81789dd8 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:14.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:14.615+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195fb00 [] 2019-12-07T19:34:14.615+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) at [0x1b000~1000] 2019-12-07T19:34:14.615+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) get_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e81789d50) put_ref 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x4c) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8441a600 0 blobs 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~26: 0x0~26 Blob(0x558e81789d50 blob([0x1b000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x26) SharedBlob(0x558e81789dc0 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.615+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_update_log_missing_reply(1.3s0 epoch 37/28 rep_tid 59 lcod 37'11) v3 -- 0x558e84065800 con 0x558e818fe400 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) dirty_range mark inline shard dirty 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:46e858f0:::pg_num_history:head# 0x0~26 = 0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_rmkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# 2019-12-07T19:34:14.615+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c1a3fc6e:::purged_snaps:0# = 0 2019-12-07T19:34:14.615+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.615+0000 7fe2d4a80700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write #-1:7b3f43c4:::osd_superblock:0# 0x0~24e - have 0x24e (590) bytes fadvise_flags 0x0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small 0x0~24e 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small considering Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) bstart 0x0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _apply_padding can pad head 0x0 tail 0xdb2 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small reading head 0x0 and tail 0x0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.BufferSpace(0x558e81078638 in 0x558e80219340) _discard 0x0~1000 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small deferred write 0x0~1000 of mutable Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) at [0x2000~1000] 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) get_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.blob(0x558e810784d0) put_ref 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x49c) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_small lex 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8441a600 0 blobs 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~24e: 0x0~24e Blob(0x558e810784d0 blob([0x2000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x24e) SharedBlob(0x558e81078620 sbid 0x0)) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) dirty_range mark inline shard dirty 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write meta #-1:7b3f43c4:::osd_superblock:0# 0x0~24e = 0 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8441a600 cost 23230 (4 ios * 4000 + 7230 bytes) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8441a600 onodes 0x558e80251b80,0x558e817b9180,0x558e84640500,0x558e84640780 shared_blobs 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update #-1:7b3f43c4:::osd_superblock:0# 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e80251c88) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:7b3f43c4:::osd_superblock:0# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update #-1:46e858f0:::pg_num_history:head# 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e817b9288) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:46e858f0:::pg_num_history:head# is 46 (21 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84640608) update #-1:c82b972f:::inc_osdmap.37:0# 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84640608) update inline shard 19 bytes from 1 extents 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:c82b972f:::inc_osdmap.37:0# is 47 (22 bytes onode + 2 bytes spanning blobs + 23 bytes inline extents) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84640888) update #-1:37b8e826:::osdmap.37:0# 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore.extentmap(0x558e84640888) update inline shard 53 bytes from 2 extents 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode #-1:37b8e826:::osdmap.37:0# is 81 (22 bytes onode + 2 bytes spanning blobs + 57 bytes inline extents) 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8441a600 allocated 0x[4bc000~3000] released 0x[] 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 freelist allocate 0x4bc000~3000 2019-12-07T19:34:14.615+0000 7fe2d4a80700 20 freelist _xor first_key 0x480000 last_key 0x480000 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441a600 prepare 2019-12-07T19:34:14.615+0000 7fe2d4a80700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8441a600 2019-12-07T19:34:14.616+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441a600 aio_wait 2019-12-07T19:34:14.616+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8441a600 2019-12-07T19:34:14.616+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441a600 io_done 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:14.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8789 < min_flush_size 524288 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e80251b80 had 1 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e817b9180 had 1 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84640500 had 1 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84640780 had 1 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c500 had 1 2019-12-07T19:34:14.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:14.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 8812 < min_flush_size 524288 2019-12-07T19:34:14.616+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.616+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x165c55~226c to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.616+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x165c55 0x165c55~226c to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~140000 x_off 0x135c55 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xc55 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xec1 and padding block with 0x13f 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x167ec1 2019-12-07T19:34:14.616+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.616+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:14.616+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000158181s (0.000004372s flush + 0.000153809s kv commit) 2019-12-07T19:34:14.616+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:14.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:14.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8441a600> 2019-12-07T19:34:14.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.616+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8441a600 kv_submitted 2019-12-07T19:34:14.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8441a600 2019-12-07T19:34:14.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _deferred_queue txc 0x558e8441a600 osr 0x558e8107a500 2019-12-07T19:34:14.616+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e84642000) _discard drop 26 0x1b000~1000 2019-12-07T19:34:14.616+0000 7fe2d7285700 10 osd.0 36 _committed_osd_maps 37..37 2019-12-07T19:34:14.616+0000 7fe2d7285700 10 osd.0 36 advance to epoch 37 (<= last 37 <= newest_map 37) 2019-12-07T19:34:14.616+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e84642000) prepare_write seq 27 0x1b000~1000 crc fc221904 2019-12-07T19:34:14.616+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e84642000) _discard drop 26 0x2000~1000 2019-12-07T19:34:14.616+0000 7fe2dca90700 20 bluestore.DeferredBatch(0x558e84642000) prepare_write seq 27 0x2000~1000 crc 3abae1df 2019-12-07T19:34:14.616+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:14.616+0000 7fe2d7285700 7 osd.0 37 consume_map version 37 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 37 identify_splits_and_merges 1.8s5 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 37 identify_splits_and_merges 1.9s3 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 37 identify_splits_and_merges 1.1s1 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 37 identify_splits_and_merges 1.as3 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 37 identify_splits_and_merges 1.2s0 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 37 identify_splits_and_merges 1.3s5 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 37 identify_splits_and_merges 1.bs1 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 37 identify_splits_and_merges 1.4s1 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 37 identify_splits_and_merges 1.5s4 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 37 identify_splits_and_merges 1.ds5 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 37 identify_splits_and_merges 1.es4 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 37 identify_splits_and_merges 1.7s3 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 37 identify_splits_and_merges 1.fs1 e36 to e37 pg_nums {18=16} 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 37 prune_pg_created 2019-12-07T19:34:14.616+0000 7fe2d7285700 10 osd.0:0.consume_map 37 (was 36) 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0:0.consume_map consume_map 1.8s5 2019-12-07T19:34:14.616+0000 7fe2d7285700 10 osd.0:1.consume_map 37 (was 36) 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.9s3 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0:1.consume_map consume_map 1.1s1 2019-12-07T19:34:14.616+0000 7fe2d7285700 10 osd.0:2.consume_map 37 (was 36) 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.as3 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0:2.consume_map consume_map 1.2s0 2019-12-07T19:34:14.616+0000 7fe2d7285700 10 osd.0:3.consume_map 37 (was 36) 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.3s5 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0:3.consume_map consume_map 1.bs1 2019-12-07T19:34:14.616+0000 7fe2d7285700 10 osd.0:4.consume_map 37 (was 36) 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0:4.consume_map consume_map 1.4s1 2019-12-07T19:34:14.616+0000 7fe2d7285700 10 osd.0:5.consume_map 37 (was 36) 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.5s4 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0:5.consume_map consume_map 1.ds5 2019-12-07T19:34:14.616+0000 7fe2d7285700 10 osd.0:6.consume_map 37 (was 36) 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0:6.consume_map consume_map 1.es4 2019-12-07T19:34:14.616+0000 7fe2d7285700 10 osd.0:7.consume_map 37 (was 36) 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.7s3 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0:7.consume_map consume_map 1.fs1 2019-12-07T19:34:14.616+0000 7fe2d7285700 10 osd.0 37 release_reserved_pushes(0), recovery_ops_reserved 0 -> 0 2019-12-07T19:34:14.616+0000 7fe2d7285700 15 osd.0 37 enqueue_peering_evt 1.8s5 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.616+0000 7fe2d7285700 15 osd.0 37 enqueue_peering_evt 1.9s3 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.616+0000 7fe2d7285700 15 osd.0 37 enqueue_peering_evt 1.1s1 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.616+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2d7285700 15 osd.0 37 enqueue_peering_evt 1.as3 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.616+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.616+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2d7285700 15 osd.0 37 enqueue_peering_evt 1.2s0 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558e8192b000 2019-12-07T19:34:14.616+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558e841a4000 2019-12-07T19:34:14.616+0000 7fe2d7285700 15 osd.0 37 enqueue_peering_evt 1.3s5 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.616+0000 7fe2d7285700 15 osd.0 37 enqueue_peering_evt 1.bs1 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.616+0000 7fe2d7285700 15 osd.0 37 enqueue_peering_evt 1.4s1 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.616+0000 7fe2d7285700 15 osd.0 37 enqueue_peering_evt 1.5s4 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.616+0000 7fe2d7285700 15 osd.0 37 enqueue_peering_evt 1.ds5 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 36 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.616+0000 7fe2cba6e700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.9s3 36 -> 37 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.616+0000 7fe2cba6e700 20 osd.0:0.update_pg_epoch 1.8s5 36 -> 37 2019-12-07T19:34:14.616+0000 7fe2d7285700 15 osd.0 37 enqueue_peering_evt 1.es4 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 36 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] handle_advance_map [1,4,5,0,7,3]/[1,4,5,0,7,3] -- 1/1 2019-12-07T19:34:14.616+0000 7fe2d7285700 15 osd.0 37 enqueue_peering_evt 1.7s3 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2cba6e700 10 osd.0 pg_epoch: 36 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] handle_advance_map [1,4,5,2,3,0]/[1,4,5,2,3,0] -- 1/1 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.616+0000 7fe2d7285700 15 osd.0 37 enqueue_peering_evt 1.fs1 epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2d7285700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.616+0000 7fe2d7285700 7 osd.0 37 activate_map version 37 2019-12-07T19:34:14.616+0000 7fe2cba6e700 10 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.616+0000 7fe2cba6e700 10 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.616+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.9s3_head options 2019-12-07T19:34:14.616+0000 7fe2cba6e700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.8s5_head options 2019-12-07T19:34:14.616+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.616+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558e81938000 2019-12-07T19:34:14.616+0000 7fe2cba6e700 10 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.616+0000 7fe2cba6e700 20 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.616+0000 7fe2cba6e700 20 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.616+0000 7fe2cba6e700 20 osd.0 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.616+0000 7fe2c5a62700 10 osd.0 pg_epoch: 36 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.616+0000 7fe2cba6e700 10 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.616+0000 7fe2c5a62700 20 osd.0:4.update_pg_epoch 1.4s1 36 -> 37 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.616+0000 7fe2cba6e700 10 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c5a62700 10 osd.0 pg_epoch: 36 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] handle_advance_map [7,0,1,5,2,4]/[7,0,1,5,2,4] -- 7/7 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2cba6e700 20 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c5a62700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2cba6e700 10 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2c5a62700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558e819e4000 2019-12-07T19:34:14.616+0000 7fe2cba6e700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.4s1_head options 2019-12-07T19:34:14.616+0000 7fe2cba6e700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.616+0000 7fe2c5a62700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.616+0000 7fe2c5a62700 20 osd.0 pg_epoch: 37 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 36 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0:1.update_pg_epoch 1.1s1 36 -> 37 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558e81231000 2019-12-07T19:34:14.616+0000 7fe2c5a62700 20 osd.0 pg_epoch: 37 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.616+0000 7fe2c5a62700 20 osd.0 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 36 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] handle_advance_map [3,0,2,4,5,1]/[3,0,2,4,5,1] -- 3/3 2019-12-07T19:34:14.616+0000 7fe2c5a62700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.616+0000 7fe2c5a62700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.3s5 36 -> 37 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.616+0000 7fe2c5a62700 20 osd.0 pg_epoch: 37 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.616+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2cb26d700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.1s1_head options 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] handle_advance_map [4,1,2,5,7,0]/[4,1,2,5,7,0] -- 4/4 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.616+0000 7fe2c5a62700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2c5a62700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.616+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.616+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.3s5_head options 2019-12-07T19:34:14.616+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558e81a98000 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 pg_epoch: 37 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c8a68700 10 osd.0 pg_epoch: 36 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 pg_epoch: 37 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.616+0000 7fe2c8a68700 20 osd.0:6.update_pg_epoch 1.es4 36 -> 37 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.616+0000 7fe2c8a68700 10 osd.0 pg_epoch: 36 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] handle_advance_map [1,3,4,5,0,2]/[1,3,4,5,0,2] -- 1/1 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c8a68700 10 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558e8193b000 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 pg_epoch: 37 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c8a68700 10 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.616+0000 7fe2c8a68700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.es4_head options 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c8a68700 10 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 36 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.as3 36 -> 37 2019-12-07T19:34:14.616+0000 7fe2c8a68700 20 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558e83c78000 2019-12-07T19:34:14.616+0000 7fe2c8a68700 20 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.616+0000 7fe2c8a68700 20 osd.0 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 36 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] handle_advance_map [1,4,2,0,5,3]/[1,4,2,0,5,3] -- 1/1 2019-12-07T19:34:14.616+0000 7fe2c8a68700 10 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.616+0000 7fe2c8a68700 10 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.616+0000 7fe2c8a68700 20 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.616+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.as3_head options 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.616+0000 7fe2c8a68700 10 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0:3.update_pg_epoch 1.bs1 36 -> 37 2019-12-07T19:34:14.616+0000 7fe2c8a68700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c8a68700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.616+0000 7fe2cc26f700 20 osd.0 37 agent_entry empty queue 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 36 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] handle_advance_map [4,0,2,7,1,5]/[4,0,2,7,1,5] -- 4/4 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2ca26b700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.bs1_head options 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558e81075000 2019-12-07T19:34:14.616+0000 7fe2ca26b700 10 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0:2.update_pg_epoch 1.2s0 36 -> 37 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558e83d07000 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 36 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] handle_advance_map [0,4,1,5,7,3]/[0,4,1,5,7,3] -- 0/0 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] state: Active advmap 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.5s4 36 -> 37 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] handle_advance_map [2,1,3,4,0,5]/[2,1,3,4,0,5] -- 2/2 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.616+0000 7fe2c6a64700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.2s0_head options 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.616+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.5s4_head options 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] agent_stop 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] activate_map 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] state: Active: handling ActMap 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] Active: kicking snap trim 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c6a64700 7 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] state: handle ActMap primary 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] prepare_stats_for_publish reporting purged_snaps [a~1] 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 op_wq(5) _process 1.ds5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c6a64700 15 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] publish_stats_to_osd 36:109 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.ds5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558e819e7000 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0:5.update_pg_epoch 1.ds5 36 -> 37 2019-12-07T19:34:14.616+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 36 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_advance_map [2,1,7,3,4,0]/[2,1,7,3,4,0] -- 2/2 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 37 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 37 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.616+0000 7fe2c9269700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.ds5_head options 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 37 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 pg_epoch: 37 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 pg_epoch: 37 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 37 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 37 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 pg_epoch: 37 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 osd.0 pg_epoch: 37 pg[1.ds5( v 21'6 (0'0,21'6] local-lis/les=18/19 n=3 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [2,1,7,3,4,0]p2(0) r=5 lpr=18 luod=0'0 crt=21'6 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2c9269700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c9269700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558e8122e000 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 36 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.7s3 36 -> 37 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 36 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] handle_advance_map [1,4,3,0,7,2]/[1,4,3,0,7,2] -- 1/1 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.616+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.7s3_head options 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 NullEvt) prio 255 cost 10 e37) pg 0x558e81a9b000 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 36 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] handle_advance_map: 37 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0:7.update_pg_epoch 1.fs1 36 -> 37 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 36 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] handle_advance_map [3,0,5,4,1,2]/[3,0,5,4,1,2] -- 3/3 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] state: Started advmap 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] check_recovery_sources no source osds () went down 2019-12-07T19:34:14.616+0000 7fe2c8267700 15 bluestore(/var/lib/ceph/osd/ceph-0) set_collection_opts 1.fs1_head options 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] plpg_on_pool_change 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] hit_set_clear 2019-12-07T19:34:14.616+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 20 ==== osd_op(client.4540.0:297 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8 ==== 302+0+4 (crc 0 0 0) 0x558e83ca6dc0 con 0x558e83d68000 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] agent_stop 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.616+0000 7fe2e6d3a700 15 osd.0 37 enqueue_op 0x558e83dd8b00 prio 63 cost 4 latency 0.000018 epoch 37 osd_op(client.4540.0:297 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.616+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:297 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8) prio 63 cost 4 e37) 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.616+0000 7fe2c8267700 10 log is not dirty 2019-12-07T19:34:14.616+0000 7fe2c8267700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:14.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:297 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8) prio 63 cost 4 e37) queued 2019-12-07T19:34:14.616+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 21 ==== osd_op(client.4540.0:298 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8 ==== 264+0+4 (crc 0 0 0) 0x558e83ca6840 con 0x558e83d68000 2019-12-07T19:34:14.616+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:297 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8) prio 63 cost 4 e37) pg 0x558e81075000 2019-12-07T19:34:14.617+0000 7fe2e6d3a700 15 osd.0 37 enqueue_op 0x558e83dd8c60 prio 63 cost 4 latency 0.000019 epoch 37 osd_op(client.4540.0:298 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8 2019-12-07T19:34:14.617+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:298 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8) prio 63 cost 4 e37) 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 37 dequeue_op 0x558e83dd8b00 prio 63 cost 4 latency 0.000051 osd_op(client.4540.0:297 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 37 maybe_share_map con 0x558e83d68000 172.21.15.131:0/224485236 map epoch 29 -> 37 (as per caller) 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] _handle_message: osd_op(client.4540.0:297 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8 2019-12-07T19:34:14.617+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4540 172.21.15.131:0/224485236 22 ==== osd_op(client.4540.0:299 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8 ==== 264+0+4 (crc 0 0 0) 0x558e83ca6580 con 0x558e83d68000 2019-12-07T19:34:14.617+0000 7fe2e6d3a700 15 osd.0 37 enqueue_op 0x558e83dd8dc0 prio 63 cost 4 latency 0.000010 epoch 37 osd_op(client.4540.0:299 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8 2019-12-07T19:34:14.617+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:299 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8) prio 63 cost 4 e37) 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op: op osd_op(client.4540.0:297 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~0,checksum in=4b,getxattrs] snapc 0=[] ondisk+read+known_if_redirected e37) v8 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op osd_op(client.4540.0:297 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~0,checksum in=4b,getxattrs] snapc 0=[] ondisk+read+known_if_redirected e37) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] get_object_context: found obc in cache: 0x558e81222900 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] get_object_context: 0x558e81222900 1:41df217a:::smithi13140796-39:head rwstate(none n=0 w=0) oi: 1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) exists: 0 ssc: 0x558e83dcc6c0 snapset: 4=[]:{} 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] find_object_context 1:41df217a:::smithi13140796-39:head @head oi=1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:34:14.617+0000 7fe2caa6c700 25 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op oi 1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op obc obc(1:41df217a:::smithi13140796-39:head(dne) rwstate(read n=1 w=0)) 2019-12-07T19:34:14.617+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(297 smithi13140796-39 [read 0~0,checksum,getxattrs] v0'0 uv0 ondisk = -2 ((2) No such file or directory)) v8 -- 0x558e84242280 con 0x558e83d68000 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 37 dequeue_op 0x558e83dd8b00 finish 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:298 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8) prio 63 cost 4 e37) queued 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:298 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8) prio 63 cost 4 e37) pg 0x558e81075000 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 37 dequeue_op 0x558e83dd8c60 prio 63 cost 4 latency 0.000176 osd_op(client.4540.0:298 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] _handle_message: osd_op(client.4540.0:298 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op: op osd_op(client.4540.0:298 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~0,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e37) v8 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op osd_op(client.4540.0:298 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~0,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e37) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] get_object_context: found obc in cache: 0x558e81222900 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] get_object_context: 0x558e81222900 1:41df217a:::smithi13140796-39:head rwstate(none n=0 w=0) oi: 1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) exists: 0 ssc: 0x558e83dcc6c0 snapset: 4=[]:{} 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] find_object_context 1:41df217a:::smithi13140796-39:head @head oi=1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:34:14.617+0000 7fe2caa6c700 25 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op oi 1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op obc obc(1:41df217a:::smithi13140796-39:head(dne) rwstate(read n=1 w=0)) 2019-12-07T19:34:14.617+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(298 smithi13140796-39 [read 0~0,checksum] v0'0 uv0 ondisk = -2 ((2) No such file or directory)) v8 -- 0x558e84242780 con 0x558e83d68000 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 37 dequeue_op 0x558e83dd8c60 finish 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:299 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8) prio 63 cost 4 e37) queued 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGOpItem(op=osd_op(client.4540.0:299 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8) prio 63 cost 4 e37) pg 0x558e81075000 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 37 dequeue_op 0x558e83dd8dc0 prio 63 cost 4 latency 0.000261 osd_op(client.4540.0:299 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8 pg pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] _handle_message: osd_op(client.4540.0:299 1.2s0 1.5e84fb82 (undecoded) ondisk+read+known_if_redirected e37) v8 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op: op osd_op(client.4540.0:299 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~0,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e37) v8 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] op_has_sufficient_caps session=0x558e818e9860 pool=1 (unique_pool_0 ) pool_app_metadata={rados={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op osd_op(client.4540.0:299 1.2s0 1:41df217a:::smithi13140796-39:head [read 0~0,checksum in=4b] snapc 0=[] ondisk+read+known_if_redirected e37) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] get_object_context: found obc in cache: 0x558e81222900 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] get_object_context: 0x558e81222900 1:41df217a:::smithi13140796-39:head rwstate(none n=0 w=0) oi: 1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) exists: 0 ssc: 0x558e83dcc6c0 snapset: 4=[]:{} 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] find_object_context 1:41df217a:::smithi13140796-39:head @head oi=1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:34:14.617+0000 7fe2caa6c700 25 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op oi 1:41df217a:::smithi13140796-39:head(0'0 unknown.0.0:0 s 0 uv 0 alloc_hint [0 0 0]) 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_op obc obc(1:41df217a:::smithi13140796-39:head(dne) rwstate(read n=1 w=0)) 2019-12-07T19:34:14.617+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/224485236 -- osd_op_reply(299 smithi13140796-39 [read 0~0,checksum] v0'0 uv0 ondisk = -2 ((2) No such file or directory)) v8 -- 0x558e84243680 con 0x558e83d68000 2019-12-07T19:34:14.617+0000 7fe2caa6c700 10 osd.0 37 dequeue_op 0x558e83dd8dc0 finish 2019-12-07T19:34:14.617+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:14.617+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:14.616+0000 7fe2ca26b700 20 osd.0 37 advance_pg new pool opts old pool opts 2019-12-07T19:34:14.627+0000 7fe2ca26b700 10 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] handle_activate_map: 37 2019-12-07T19:34:14.627+0000 7fe2ca26b700 10 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] activate_map 2019-12-07T19:34:14.627+0000 7fe2ca26b700 20 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] activate_map: Not dirtying info: last_persisted is 36 while current is 37 2019-12-07T19:34:14.627+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:14.627+0000 7fe2ca26b700 10 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 NullEvt 2019-12-07T19:34:14.627+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:14.627+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.629+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 138 ==== MOSDECSubOpWrite(1.3s5 37/28 ECSubWrite(tid=60, reqid=client.4540.0:292, at_version=37'13, trim_to=0'0, roll_forward_to=23'10)) v2 ==== 158902+0+0 (unknown 0 0 0) 0x558e83e55000 con 0x558e818fe400 2019-12-07T19:34:14.629+0000 7fe2e6d3a700 15 osd.0 37 enqueue_op 0x558e83dd8f20 prio 127 cost 0 latency 0.000043 epoch 37 MOSDECSubOpWrite(1.3s5 37/28 ECSubWrite(tid=60, reqid=client.4540.0:292, at_version=37'13, trim_to=0'0, roll_forward_to=23'10)) v2 2019-12-07T19:34:14.629+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 37/28 ECSubWrite(tid=60, reqid=client.4540.0:292, at_version=37'13, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 37/28 ECSubWrite(tid=60, reqid=client.4540.0:292, at_version=37'13, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 37/28 ECSubWrite(tid=60, reqid=client.4540.0:292, at_version=37'13, trim_to=0'0, roll_forward_to=23'10)) v2) prio 127 cost 0 e37) pg 0x558e81231000 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 osd.0 37 dequeue_op 0x558e83dd8f20 prio 127 cost 0 latency 0.001160 MOSDECSubOpWrite(1.3s5 37/28 ECSubWrite(tid=60, reqid=client.4540.0:292, at_version=37'13, trim_to=0'0, roll_forward_to=23'10)) v2 pg pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.3s5 37/28 ECSubWrite(tid=60, reqid=client.4540.0:292, at_version=37'13, trim_to=0'0, roll_forward_to=23'10)) v2 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 snap_mapper.add_oid 1:c8ce3553:::smithi13140796-29:e c,d,e 2019-12-07T19:34:14.630+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 snap_mapper.get_snaps 1:c8ce3553:::smithi13140796-29:e got.empty() 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 snap_mapper.set_snaps 1:c8ce3553:::smithi13140796-29:e c,d,e 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 snap_mapper.set_snaps set OBJ_.5_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.5_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 snap_mapper.add_oid set SNA_1_000000000000000D_.5_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.5_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'11 (0'0,37'11] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] append_log log((0'0,37'11], crt=23'10) [37'12 (21'6) clone 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:33:58.727310+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,37'13 (21'6) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:292 0.000000 0 ObjectCleanRegions clean_offsets: [622592~18446744073708929023], clean_omap: 1, new_object: 0] 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'12 (0'0,37'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 lua=37'11 crt=23'10 lcod 37'11 active mbc={} ps=[a~1]] add_log_entry 37'12 (21'6) clone 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:33:58.727310+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 lua=37'11 crt=23'10 lcod 37'11 active mbc={} ps=[a~1]] add_log_entry 37'13 (21'6) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:292 0.000000 0 ObjectCleanRegions clean_offsets: [622592~18446744073708929023], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 lua=37'11 crt=23'10 lcod 37'11 active mbc={} ps=[a~1]] append_log approx pg log length = 13 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 lua=37'11 crt=23'10 lcod 37'11 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.630+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 37'12, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e8195e900 seq 20 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) get_onode oid 5#1:c8ce3553:::smithi13140796-29:e# key 0x858000000000000001c8ce35'S!smithi13140796-29!='0x000000000000000effffffffffffffff'o' 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) r -2 v.len 0 2019-12-07T19:34:14.630+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _clone 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# -> 5#1:c8ce3553:::smithi13140796-29:e# 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1169 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.onode(0x558e84242c80).flush flush done 2019-12-07T19:34:14.630+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:e# 0x0 2019-12-07T19:34:14.630+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_clone_range 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# -> 5#1:c8ce3553:::smithi13140796-29:e# 0x0~1d000 -> 0x0~1d000 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0x0~2000: 0x0~2000 Blob(0x558e841a2930 blob([0x378000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a28c0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10283 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e841a2930 blob([0x378000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a28c0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e841a2930 blob([0x378000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a28c0 loaded (sbid 0x282b ref_map(0x378000~2000=1)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e81a49030 blob([0x378000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a28c0 loaded (sbid 0x282b ref_map(0x378000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49030) get_ref 0x0~2000 Blob(0x558e81a49030 blob([0x378000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841a28c0 loaded (sbid 0x282b ref_map(0x378000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49030) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0x0~2000: 0x0~2000 Blob(0x558e81a49030 blob([0x378000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a28c0 loaded (sbid 0x282b ref_map(0x378000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0x2000~2000: 0x0~2000 Blob(0x558e841a2850 blob([0x37a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f8f0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10284 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e841a2850 blob([0x37a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f8f0 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e841a2850 blob([0x37a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f8f0 loaded (sbid 0x282c ref_map(0x37a000~2000=1)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e81a48fc0 blob([0x37a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f8f0 loaded (sbid 0x282c ref_map(0x37a000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.blob(0x558e81a48fc0) get_ref 0x0~2000 Blob(0x558e81a48fc0 blob([0x37a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f8f0 loaded (sbid 0x282c ref_map(0x37a000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.blob(0x558e81a48fc0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0x2000~2000: 0x0~2000 Blob(0x558e81a48fc0 blob([0x37a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f8f0 loaded (sbid 0x282c ref_map(0x37a000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0x4000~2000: 0x0~2000 Blob(0x558e83d0f1f0 blob([0x37c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f180 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10285 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e83d0f1f0 blob([0x37c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f180 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e83d0f1f0 blob([0x37c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f180 loaded (sbid 0x282d ref_map(0x37c000~2000=1)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e83e6c8c0 blob([0x37c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f180 loaded (sbid 0x282d ref_map(0x37c000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6c8c0) get_ref 0x0~2000 Blob(0x558e83e6c8c0 blob([0x37c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0f180 loaded (sbid 0x282d ref_map(0x37c000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6c8c0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0x4000~2000: 0x0~2000 Blob(0x558e83e6c8c0 blob([0x37c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f180 loaded (sbid 0x282d ref_map(0x37c000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0x6000~2000: 0x0~2000 Blob(0x558e83af90a0 blob([0x37e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abd570 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10286 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e83af90a0 blob([0x37e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abd570 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e83af90a0 blob([0x37e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abd570 loaded (sbid 0x282e ref_map(0x37e000~2000=1)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e81a7bb20 blob([0x37e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abd570 loaded (sbid 0x282e ref_map(0x37e000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7bb20) get_ref 0x0~2000 Blob(0x558e81a7bb20 blob([0x37e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83abd570 loaded (sbid 0x282e ref_map(0x37e000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7bb20) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0x6000~2000: 0x0~2000 Blob(0x558e81a7bb20 blob([0x37e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abd570 loaded (sbid 0x282e ref_map(0x37e000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0x8000~2000: 0x0~2000 Blob(0x558e83e6fea0 blob([0x380000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6f420 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10287 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e83e6fea0 blob([0x380000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6f420 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e83e6fea0 blob([0x380000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6f420 loaded (sbid 0x282f ref_map(0x380000~2000=1)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e81a490a0 blob([0x380000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f420 loaded (sbid 0x282f ref_map(0x380000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.blob(0x558e81a490a0) get_ref 0x0~2000 Blob(0x558e81a490a0 blob([0x380000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83e6f420 loaded (sbid 0x282f ref_map(0x380000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.blob(0x558e81a490a0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0x8000~2000: 0x0~2000 Blob(0x558e81a490a0 blob([0x380000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6f420 loaded (sbid 0x282f ref_map(0x380000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0xa000~2000: 0x0~2000 Blob(0x558e83e6f650 blob([0x382000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2d20 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10288 2019-12-07T19:34:14.630+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e83e6f650 blob([0x382000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2d20 sbid 0x0)) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e83e6f650 blob([0x382000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2d20 loaded (sbid 0x2830 ref_map(0x382000~2000=1)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e81a49110 blob([0x382000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2d20 loaded (sbid 0x2830 ref_map(0x382000~2000=2)))) 2019-12-07T19:34:14.630+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49110) get_ref 0x0~2000 Blob(0x558e81a49110 blob([0x382000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2d20 loaded (sbid 0x2830 ref_map(0x382000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49110) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0xa000~2000: 0x0~2000 Blob(0x558e81a49110 blob([0x382000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2d20 loaded (sbid 0x2830 ref_map(0x382000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0xc000~2000: 0x0~2000 Blob(0x558e83dd2690 blob([0x384000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2460 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10289 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e83dd2690 blob([0x384000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2460 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e83dd2690 blob([0x384000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2460 loaded (sbid 0x2831 ref_map(0x384000~2000=1)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e81a491f0 blob([0x384000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2460 loaded (sbid 0x2831 ref_map(0x384000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a491f0) get_ref 0x0~2000 Blob(0x558e81a491f0 blob([0x384000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83dd2460 loaded (sbid 0x2831 ref_map(0x384000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a491f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0xc000~2000: 0x0~2000 Blob(0x558e81a491f0 blob([0x384000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2460 loaded (sbid 0x2831 ref_map(0x384000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0xe000~2000: 0x0~2000 Blob(0x558e83edefc0 blob([0x386000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ccb0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10290 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e83edefc0 blob([0x386000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ccb0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e83edefc0 blob([0x386000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ccb0 loaded (sbid 0x2832 ref_map(0x386000~2000=1)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e81a49260 blob([0x386000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423ccb0 loaded (sbid 0x2832 ref_map(0x386000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49260) get_ref 0x0~2000 Blob(0x558e81a49260 blob([0x386000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8423ccb0 loaded (sbid 0x2832 ref_map(0x386000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49260) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0xe000~2000: 0x0~2000 Blob(0x558e81a49260 blob([0x386000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ccb0 loaded (sbid 0x2832 ref_map(0x386000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0x10000~2000: 0x0~2000 Blob(0x558e83d02540 blob([0x388000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d029a0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10291 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e83d02540 blob([0x388000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d029a0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e83d02540 blob([0x388000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d029a0 loaded (sbid 0x2833 ref_map(0x388000~2000=1)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e81a492d0 blob([0x388000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d029a0 loaded (sbid 0x2833 ref_map(0x388000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a492d0) get_ref 0x0~2000 Blob(0x558e81a492d0 blob([0x388000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d029a0 loaded (sbid 0x2833 ref_map(0x388000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a492d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0x10000~2000: 0x0~2000 Blob(0x558e81a492d0 blob([0x388000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d029a0 loaded (sbid 0x2833 ref_map(0x388000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0x12000~2000: 0x0~2000 Blob(0x558e841dbc00 blob([0x38a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbb90 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10292 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e841dbc00 blob([0x38a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbb90 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e841dbc00 blob([0x38a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbb90 loaded (sbid 0x2834 ref_map(0x38a000~2000=1)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e81a49340 blob([0x38a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbb90 loaded (sbid 0x2834 ref_map(0x38a000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49340) get_ref 0x0~2000 Blob(0x558e81a49340 blob([0x38a000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbb90 loaded (sbid 0x2834 ref_map(0x38a000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49340) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0x12000~2000: 0x0~2000 Blob(0x558e81a49340 blob([0x38a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbb90 loaded (sbid 0x2834 ref_map(0x38a000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0x14000~2000: 0x0~2000 Blob(0x558e841dbb20 blob([0x38c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbab0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10293 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e841dbb20 blob([0x38c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbab0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e841dbb20 blob([0x38c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbab0 loaded (sbid 0x2835 ref_map(0x38c000~2000=1)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e81a493b0 blob([0x38c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbab0 loaded (sbid 0x2835 ref_map(0x38c000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a493b0) get_ref 0x0~2000 Blob(0x558e81a493b0 blob([0x38c000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841dbab0 loaded (sbid 0x2835 ref_map(0x38c000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a493b0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0x14000~2000: 0x0~2000 Blob(0x558e81a493b0 blob([0x38c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbab0 loaded (sbid 0x2835 ref_map(0x38c000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0x16000~2000: 0x0~2000 Blob(0x558e841dba40 blob([0x38e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db9d0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10294 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e841dba40 blob([0x38e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db9d0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e841dba40 blob([0x38e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db9d0 loaded (sbid 0x2836 ref_map(0x38e000~2000=1)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e81a49420 blob([0x38e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db9d0 loaded (sbid 0x2836 ref_map(0x38e000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49420) get_ref 0x0~2000 Blob(0x558e81a49420 blob([0x38e000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db9d0 loaded (sbid 0x2836 ref_map(0x38e000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49420) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0x16000~2000: 0x0~2000 Blob(0x558e81a49420 blob([0x38e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db9d0 loaded (sbid 0x2836 ref_map(0x38e000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0x18000~2000: 0x0~2000 Blob(0x558e841db960 blob([0x390000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db8f0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10295 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e841db960 blob([0x390000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db8f0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e841db960 blob([0x390000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db8f0 loaded (sbid 0x2837 ref_map(0x390000~2000=1)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e81a49490 blob([0x390000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db8f0 loaded (sbid 0x2837 ref_map(0x390000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49490) get_ref 0x0~2000 Blob(0x558e81a49490 blob([0x390000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e841db8f0 loaded (sbid 0x2837 ref_map(0x390000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49490) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0x18000~2000: 0x0~2000 Blob(0x558e81a49490 blob([0x390000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db8f0 loaded (sbid 0x2837 ref_map(0x390000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0x1a000~2000: 0x0~2000 Blob(0x558e841db880 blob([0x392000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9570 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10296 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e841db880 blob([0x392000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9570 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e841db880 blob([0x392000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9570 loaded (sbid 0x2838 ref_map(0x392000~2000=1)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e81a49500 blob([0x392000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9570 loaded (sbid 0x2838 ref_map(0x392000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49500) get_ref 0x0~2000 Blob(0x558e81a49500 blob([0x392000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e817e9570 loaded (sbid 0x2838 ref_map(0x392000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49500) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0x1a000~2000: 0x0~2000 Blob(0x558e81a49500 blob([0x392000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9570 loaded (sbid 0x2838 ref_map(0x392000~2000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup src 0x1c000~1000: 0x0~1000 Blob(0x558e817e9730 blob([0x394000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d37f80 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10297 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared Blob(0x558e817e9730 blob([0x394000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d37f80 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) make_blob_shared now Blob(0x558e817e9730 blob([0x394000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d37f80 loaded (sbid 0x2839 ref_map(0x394000~1000=1)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup new Blob(0x558e81a49570 blob([0x394000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d37f80 loaded (sbid 0x2839 ref_map(0x394000~1000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49570) get_ref 0x0~1000 Blob(0x558e81a49570 blob([0x394000~1000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d37f80 loaded (sbid 0x2839 ref_map(0x394000~1000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49570) get_ref init 0x1000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dup dst 0x1c000~1000: 0x0~1000 Blob(0x558e81a49570 blob([0x394000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d37f80 loaded (sbid 0x2839 ref_map(0x394000~1000=2)))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dirty_range mark inline shard dirty 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e846c3008) dirty_range mark inline shard dirty 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _clone 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# -> 5#1:c8ce3553:::smithi13140796-29:e# = 0 2019-12-07T19:34:14.631+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rmattr 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:e# snapset 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rmattr 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:e# snapset = 0 2019-12-07T19:34:14.631+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:e# 1 keys 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:e# 1 keys = 0 2019-12-07T19:34:14.631+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:e# hinfo_key (42 bytes) 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) get_onode oid 5#1:c8ce3553:::smithi13140796-29:head#d key 0x858000000000000001c8ce35'S!smithi13140796-29!='0xfffffffffffffffe000000000000000d'o' 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.3s5_head 0x558e8403f2c0) r -2 v.len 0 2019-12-07T19:34:14.631+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# -> 5#1:c8ce3553:::smithi13140796-29:head#d 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# -> 5#1:c8ce3553:::smithi13140796-29:head#d = 0 2019-12-07T19:34:14.631+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1170 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# = 0 2019-12-07T19:34:14.631+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 3 keys 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 3 keys = 0 2019-12-07T19:34:14.631+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:14.631+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 0x0~26000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 5#1:c8ce3553:::smithi13140796-29:head# 0x0~26000 - have 0x0 (0) bytes fadvise_flags 0x20 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~26000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e8195e900 19 blobs 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate 0x26000/1000,26000,0 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x4bf000~1b000/1000,26000,0 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x537000~b000/1000,26000,0 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x4bf000~1b000,0x537000~b000] 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a49650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b8f0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a49650 blob([0x4bf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b8f0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49650) get_ref 0x0~2000 Blob(0x558e81a49650 blob([0x4bf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a7b8f0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49650) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e81a49650 blob([0x4bf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a7b8f0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a7b908 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a7b880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84014a10 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a7b880 blob([0x4c1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84014a10 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7b880) get_ref 0x0~2000 Blob(0x558e81a7b880 blob([0x4c1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84014a10 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a7b880) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e81a7b880 blob([0x4c1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84014a10 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84014a28 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e84190070 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a496c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e84190070 blob([0x4c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a496c0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e84190070) get_ref 0x0~2000 Blob(0x558e84190070 blob([0x4c3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a496c0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e84190070) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e84190070 blob([0x4c3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a496c0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a496d8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a49730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a497a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a49730 blob([0x4c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a497a0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49730) get_ref 0x0~2000 Blob(0x558e81a49730 blob([0x4c5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a497a0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49730) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e81a49730 blob([0x4c5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a497a0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a497b8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a49810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a49880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a49810 blob([0x4c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49880 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49810) get_ref 0x0~2000 Blob(0x558e81a49810 blob([0x4c7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49880 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49810) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e81a49810 blob([0x4c7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a49880 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a49898 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a498f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a49960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a498f0 blob([0x4c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49960 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a498f0) get_ref 0x0~2000 Blob(0x558e81a498f0 blob([0x4c9000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49960 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a498f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e81a498f0 blob([0x4c9000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a49960 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a49978 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a499d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a49a40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a499d0 blob([0x4cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49a40 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a499d0) get_ref 0x0~2000 Blob(0x558e81a499d0 blob([0x4cb000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49a40 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a499d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e81a499d0 blob([0x4cb000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a49a40 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a49a58 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a49ab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e81a49b20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a49ab0 blob([0x4cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49b20 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49ab0) get_ref 0x0~2000 Blob(0x558e81a49ab0 blob([0x4cd000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e81a49b20 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49ab0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e81a49ab0 blob([0x4cd000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e81a49b20 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e81a49b38 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81a49b90 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84014f50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81a49b90 blob([0x4cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84014f50 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49b90) get_ref 0x0~2000 Blob(0x558e81a49b90 blob([0x4cf000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84014f50 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81a49b90) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e81a49b90 blob([0x4cf000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84014f50 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84014f68 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c5a40 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d02460 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c5a40 blob([0x4d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02460 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e846c5a40) get_ref 0x0~2000 Blob(0x558e846c5a40 blob([0x4d1000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02460 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e846c5a40) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~2000: 0x0~2000 Blob(0x558e846c5a40 blob([0x4d1000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02460 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d02478 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e83d024d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d02bd0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e83d024d0 blob([0x4d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02bd0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e83d024d0) get_ref 0x0~2000 Blob(0x558e83d024d0 blob([0x4d3000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d02bd0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e83d024d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x14000~2000: 0x0~2000 Blob(0x558e83d024d0 blob([0x4d3000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d02bd0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d02be8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c59d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e811d09a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c59d0 blob([0x4d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d09a0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e846c59d0) get_ref 0x0~2000 Blob(0x558e846c59d0 blob([0x4d5000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e811d09a0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e846c59d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x16000~2000: 0x0~2000 Blob(0x558e846c59d0 blob([0x4d5000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e811d09a0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e811d09b8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e81867490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84030e00 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e81867490 blob([0x4d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84030e00 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81867490) get_ref 0x0~2000 Blob(0x558e81867490 blob([0x4d7000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84030e00 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e81867490) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x18000~2000: 0x0~2000 Blob(0x558e81867490 blob([0x4d7000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84030e00 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84030e18 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c5960 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194e7e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c5960 blob([0x4d9000~1000,0x537000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e7e0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e846c5960) get_ref 0x0~2000 Blob(0x558e846c5960 blob([0x4d9000~1000,0x537000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194e7e0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e846c5960) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1a000~2000: 0x0~2000 Blob(0x558e846c5960 blob([0x4d9000~1000,0x537000~1000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194e7e0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e8194e7f8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e811d1880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e840310a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e811d1880 blob([0x538000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840310a0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e811d1880) get_ref 0x0~2000 Blob(0x558e811d1880 blob([0x538000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e840310a0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e811d1880) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1c000~2000: 0x0~2000 Blob(0x558e811d1880 blob([0x538000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e840310a0 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e840310b8 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c58f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8194fb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c58f0 blob([0x53a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194fb20 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e846c58f0) get_ref 0x0~2000 Blob(0x558e846c58f0 blob([0x53a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194fb20 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e846c58f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x1e000~2000: 0x0~2000 Blob(0x558e846c58f0 blob([0x53a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194fb20 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e8194fb38 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194f9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e84030a80 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194f9d0 blob([0x53c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84030a80 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e8194f9d0) get_ref 0x0~2000 Blob(0x558e8194f9d0 blob([0x53c000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e84030a80 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e8194f9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x20000~2000: 0x0~2000 Blob(0x558e8194f9d0 blob([0x53c000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e84030a80 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e84030a98 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e846c5880 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fd50 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e846c5880 blob([0x53e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fd50 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e846c5880) get_ref 0x0~2000 Blob(0x558e846c5880 blob([0x53e000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d0fd50 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e846c5880) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x22000~2000: 0x0~2000 Blob(0x558e846c5880 blob([0x53e000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0fd50 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83d0fd68 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8194fab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e83edf650 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8194fab0 blob([0x540000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf650 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e8194fab0) get_ref 0x0~2000 Blob(0x558e8194fab0 blob([0x540000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83edf650 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.blob(0x558e8194fab0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x24000~2000: 0x0~2000 Blob(0x558e8194fab0 blob([0x540000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83edf650 sbid 0x0)) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.BufferSpace(0x558e83edf668 in 0x558e802196c0) _discard 0x0~2000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x26000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84641508) dirty_range mark inline shard dirty 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# 0x0~26000 = 0 2019-12-07T19:34:14.631+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.631+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.5_0000000000000001.3137CAAC.e.smithi13140796-29..' <- OBJ_.5_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.631+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.5_0000000000000001.3137CAAC.e.smithi13140796-29..' <- SNA_1_000000000000000C_.5_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.5_0000000000000001.3137CAAC.e.smithi13140796-29..' <- SNA_1_000000000000000D_.5_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.5_0000000000000001.3137CAAC.e.smithi13140796-29..' <- SNA_1_000000000000000E_.5_0000000000000001.3137CAAC.e.smithi13140796-29.. 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.631+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.0000000037.00000000000000000012' <- 0000000037.00000000000000000012 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.0000000037.00000000000000000013' <- 0000000037.00000000000000000013 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._epoch' <- _epoch 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e8195e900 cost 244265 (21 ios * 4000 + 160265 bytes) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e8195e900 onodes 0x558e84242c80,0x558e84641400,0x558e846c2f00 shared_blobs 0x558e817e9570,0x558e83abd570,0x558e83d029a0,0x558e83d0f180,0x558e83d0f8f0,0x558e83d37f80,0x558e83dd2460,0x558e83dd2d20,0x558e83e6f420,0x558e841a28c0,0x558e841db8f0,0x558e841db9d0,0x558e841dbab0,0x558e841dbb90,0x558e8423ccb0 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) update 5#1:c8ce3553:::smithi13140796-29:head#d 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) update inline shard 405 bytes from 15 extents 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:c8ce3553:::smithi13140796-29:head#d is 896 (485 bytes onode + 2 bytes spanning blobs + 409 bytes inline extents) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84641508) update 5#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84641508) update inline shard 369 bytes from 19 extents 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:c8ce3553:::smithi13140796-29:head# is 932 (557 bytes onode + 2 bytes spanning blobs + 373 bytes inline extents) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e846c3008) update 5#1:c8ce3553:::smithi13140796-29:e# 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore.extentmap(0x558e846c3008) update inline shard 405 bytes from 15 extents 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 5#1:c8ce3553:::smithi13140796-29:e# is 823 (412 bytes onode + 2 bytes spanning blobs + 409 bytes inline extents) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2838 is 11 SharedBlob(0x558e817e9570 loaded (sbid 0x2838 ref_map(0x392000~2000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x282e is 11 SharedBlob(0x558e83abd570 loaded (sbid 0x282e ref_map(0x37e000~2000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2833 is 11 SharedBlob(0x558e83d029a0 loaded (sbid 0x2833 ref_map(0x388000~2000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x282d is 11 SharedBlob(0x558e83d0f180 loaded (sbid 0x282d ref_map(0x37c000~2000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x282c is 11 SharedBlob(0x558e83d0f8f0 loaded (sbid 0x282c ref_map(0x37a000~2000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2839 is 11 SharedBlob(0x558e83d37f80 loaded (sbid 0x2839 ref_map(0x394000~1000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2831 is 11 SharedBlob(0x558e83dd2460 loaded (sbid 0x2831 ref_map(0x384000~2000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2830 is 11 SharedBlob(0x558e83dd2d20 loaded (sbid 0x2830 ref_map(0x382000~2000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x282f is 11 SharedBlob(0x558e83e6f420 loaded (sbid 0x282f ref_map(0x380000~2000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x282b is 11 SharedBlob(0x558e841a28c0 loaded (sbid 0x282b ref_map(0x378000~2000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2837 is 11 SharedBlob(0x558e841db8f0 loaded (sbid 0x2837 ref_map(0x390000~2000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2836 is 11 SharedBlob(0x558e841db9d0 loaded (sbid 0x2836 ref_map(0x38e000~2000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2835 is 11 SharedBlob(0x558e841dbab0 loaded (sbid 0x2835 ref_map(0x38c000~2000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2834 is 11 SharedBlob(0x558e841dbb90 loaded (sbid 0x2834 ref_map(0x38a000~2000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2832 is 11 SharedBlob(0x558e8423ccb0 loaded (sbid 0x2832 ref_map(0x386000~2000=2))) 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e8195e900 allocated 0x[4bf000~1b000,537000~b000] released 0x[] 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 freelist allocate 0x4bf000~1b000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 freelist _xor first_key 0x480000 last_key 0x480000 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 freelist allocate 0x537000~b000 2019-12-07T19:34:14.631+0000 7fe2c6263700 20 freelist _xor first_key 0x500000 last_key 0x500000 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 prepare 2019-12-07T19:34:14.631+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e8195e900 2019-12-07T19:34:14.632+0000 7fe2c6263700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 lua=37'11 crt=23'10 lcod 37'11 active mbc={} ps=[a~1]] op_applied version 37'13 2019-12-07T19:34:14.632+0000 7fe2c6263700 10 osd.0 37 dequeue_op 0x558e83dd8f20 finish 2019-12-07T19:34:14.632+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.632+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 aio_wait 2019-12-07T19:34:14.632+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e8195e900 2019-12-07T19:34:14.632+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 io_done 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:14.632+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 319 < min_flush_size 524288 2019-12-07T19:34:14.632+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5764 < min_flush_size 524288 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84242c80 had 1 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84641400 had 1 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e846c2f00 had 1 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:14.632+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5787 < min_flush_size 524288 2019-12-07T19:34:14.632+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5787 < min_flush_size 524288 2019-12-07T19:34:14.632+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.632+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.632+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x167ec1~169b to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.632+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x167ec1 0x167ec1~169b to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~140000 x_off 0x137ec1 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xec1 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x55c and padding block with 0xaa4 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x16955c 2019-12-07T19:34:14.632+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.632+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:14.632+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000206739s (0.000005099s flush + 0.000201640s kv commit) 2019-12-07T19:34:14.632+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e8195e900> 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.632+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 kv_submitted 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e8195e900 2019-12-07T19:34:14.632+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e8195e900 finishing 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e8195e900 onodes 0x558e84242c80,0x558e84641400,0x558e846c2f00 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e811d09b8 in 0x558e802196c0) _finish_write discard buffer(0x558e84246360 space 0x558e811d09b8 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194e7f8 in 0x558e802196c0) _finish_write discard buffer(0x558e84246480 space 0x558e8194e7f8 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8194fb38 in 0x558e802196c0) _finish_write discard buffer(0x558e842465a0 space 0x558e8194fb38 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a496d8 in 0x558e802196c0) _finish_write discard buffer(0x558e843ba900 space 0x558e81a496d8 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a497b8 in 0x558e802196c0) _finish_write discard buffer(0x558e843ba750 space 0x558e81a497b8 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a49898 in 0x558e802196c0) _finish_write discard buffer(0x558e843ba7e0 space 0x558e81a49898 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a49978 in 0x558e802196c0) _finish_write discard buffer(0x558e84246000 space 0x558e81a49978 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a49a58 in 0x558e802196c0) _finish_write discard buffer(0x558e84246090 space 0x558e81a49a58 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a49b38 in 0x558e802196c0) _finish_write discard buffer(0x558e84246120 space 0x558e81a49b38 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e81a7b908 in 0x558e802196c0) _finish_write discard buffer(0x558e843bad80 space 0x558e81a7b908 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d02478 in 0x558e802196c0) _finish_write discard buffer(0x558e84246240 space 0x558e83d02478 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d02be8 in 0x558e802196c0) _finish_write discard buffer(0x558e842462d0 space 0x558e83d02be8 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83d0fd68 in 0x558e802196c0) _finish_write discard buffer(0x558e842466c0 space 0x558e83d0fd68 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e83edf668 in 0x558e802196c0) _finish_write discard buffer(0x558e84246750 space 0x558e83edf668 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84014a28 in 0x558e802196c0) _finish_write discard buffer(0x558e843ba6c0 space 0x558e84014a28 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84014f68 in 0x558e802196c0) _finish_write discard buffer(0x558e842461b0 space 0x558e84014f68 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84030a98 in 0x558e802196c0) _finish_write discard buffer(0x558e84246630 space 0x558e84030a98 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e84030e18 in 0x558e802196c0) _finish_write discard buffer(0x558e842463f0 space 0x558e84030e18 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e840310b8 in 0x558e802196c0) _finish_write discard buffer(0x558e84246510 space 0x558e840310b8 0x0~2000 writing nocache) 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e8195e900 done 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:14.632+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e8195e900 [] 2019-12-07T19:34:14.632+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:14.632+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:14.632+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.632+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 37/28 ECSubWriteReply(tid=60, last_complete=37'13, committed=1, applied=1)) v2 -- 0x558e818eea00 con 0x558e818fe400 2019-12-07T19:34:14.632+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.633+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 139 ==== MOSDECSubOpWrite(1.3s5 37/28 ECSubWrite(tid=61, reqid=client.4540.0:292, at_version=0'0, trim_to=0'0, roll_forward_to=37'13)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e83e54000 con 0x558e818fe400 2019-12-07T19:34:14.633+0000 7fe2e6d3a700 15 osd.0 37 enqueue_op 0x558e83dd9080 prio 127 cost 0 latency 0.000025 epoch 37 MOSDECSubOpWrite(1.3s5 37/28 ECSubWrite(tid=61, reqid=client.4540.0:292, at_version=0'0, trim_to=0'0, roll_forward_to=37'13)) v2 2019-12-07T19:34:14.633+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 37/28 ECSubWrite(tid=61, reqid=client.4540.0:292, at_version=0'0, trim_to=0'0, roll_forward_to=37'13)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 37/28 ECSubWrite(tid=61, reqid=client.4540.0:292, at_version=0'0, trim_to=0'0, roll_forward_to=37'13)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGOpItem(op=MOSDECSubOpWrite(1.3s5 37/28 ECSubWrite(tid=61, reqid=client.4540.0:292, at_version=0'0, trim_to=0'0, roll_forward_to=37'13)) v2) prio 127 cost 0 e37) pg 0x558e81231000 2019-12-07T19:34:14.633+0000 7fe2c6263700 10 osd.0 37 dequeue_op 0x558e83dd9080 prio 127 cost 0 latency 0.000060 MOSDECSubOpWrite(1.3s5 37/28 ECSubWrite(tid=61, reqid=client.4540.0:292, at_version=0'0, trim_to=0'0, roll_forward_to=37'13)) v2 pg pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.633+0000 7fe2c6263700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.3s5 37/28 ECSubWrite(tid=61, reqid=client.4540.0:292, at_version=0'0, trim_to=0'0, roll_forward_to=37'13)) v2 2019-12-07T19:34:14.633+0000 7fe2c6263700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=23'10 active mbc={} ps=[a~1]] append_log log((0'0,37'13], crt=23'10) [] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] rollforward: entry=37'11 (0'0) error 1:c87708e9:::smithi13140796-17:head by client.4540.0:288 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] rollforward: entry=37'12 (21'6) clone 1:c8ce3553:::smithi13140796-29:e by unknown.0.0:0 2019-12-07T19:33:58.727310+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] rollforward: entry=37'13 (21'6) modify 1:c8ce3553:::smithi13140796-29:head by client.4540.0:292 0.000000 0 ObjectCleanRegions clean_offsets: [622592~18446744073708929023], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.633+0000 7fe2c6263700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] append_log approx pg log length = 13 2019-12-07T19:34:14.633+0000 7fe2c6263700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.633+0000 7fe2c6263700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.633+0000 7fe2c6263700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.633+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e8403f2c0 1.3s5_head 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e83c6f2c0 = 0x558e83e73800 seq 21 2019-12-07T19:34:14.633+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head#d onode 0x558e84242c80 txc 0x558e83e73800 2019-12-07T19:34:14.633+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head#d 0x0 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e841a2930) put_ref 0x0~2000 Blob(0x558e841a2930 blob([0x378000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841a28c0 loaded (sbid 0x282b ref_map(0x378000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e841a2850) put_ref 0x0~2000 Blob(0x558e841a2850 blob([0x37a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f8f0 loaded (sbid 0x282c ref_map(0x37a000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e83d0f1f0) put_ref 0x0~2000 Blob(0x558e83d0f1f0 blob([0x37c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d0f180 loaded (sbid 0x282d ref_map(0x37c000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e83af90a0) put_ref 0x0~2000 Blob(0x558e83af90a0 blob([0x37e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83abd570 loaded (sbid 0x282e ref_map(0x37e000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6fea0) put_ref 0x0~2000 Blob(0x558e83e6fea0 blob([0x380000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83e6f420 loaded (sbid 0x282f ref_map(0x380000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e83e6f650) put_ref 0x0~2000 Blob(0x558e83e6f650 blob([0x382000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2d20 loaded (sbid 0x2830 ref_map(0x382000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e83dd2690) put_ref 0x0~2000 Blob(0x558e83dd2690 blob([0x384000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83dd2460 loaded (sbid 0x2831 ref_map(0x384000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e83edefc0) put_ref 0x0~2000 Blob(0x558e83edefc0 blob([0x386000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8423ccb0 loaded (sbid 0x2832 ref_map(0x386000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e83d02540) put_ref 0x0~2000 Blob(0x558e83d02540 blob([0x388000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d029a0 loaded (sbid 0x2833 ref_map(0x388000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e841dbc00) put_ref 0x0~2000 Blob(0x558e841dbc00 blob([0x38a000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbb90 loaded (sbid 0x2834 ref_map(0x38a000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e841dbb20) put_ref 0x0~2000 Blob(0x558e841dbb20 blob([0x38c000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841dbab0 loaded (sbid 0x2835 ref_map(0x38c000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e841dba40) put_ref 0x0~2000 Blob(0x558e841dba40 blob([0x38e000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db9d0 loaded (sbid 0x2836 ref_map(0x38e000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e841db960) put_ref 0x0~2000 Blob(0x558e841db960 blob([0x390000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e841db8f0 loaded (sbid 0x2837 ref_map(0x390000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e841db880) put_ref 0x0~2000 Blob(0x558e841db880 blob([0x392000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e817e9570 loaded (sbid 0x2838 ref_map(0x392000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.blob(0x558e817e9730) put_ref 0x0~1000 Blob(0x558e817e9730 blob([0x394000~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e83d37f80 loaded (sbid 0x2839 ref_map(0x394000~1000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore.extentmap(0x558e84242d88) dirty_range mark inline shard dirty 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e841a2930 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e841a28c0 loaded (sbid 0x282b ref_map(0x378000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x378000~2000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e841a28c0 loaded (sbid 0x282b ref_map(0x378000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e841a2850 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d0f8f0 loaded (sbid 0x282c ref_map(0x37a000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x37a000~2000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83d0f8f0 loaded (sbid 0x282c ref_map(0x37a000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e83d0f1f0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d0f180 loaded (sbid 0x282d ref_map(0x37c000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x37c000~2000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83d0f180 loaded (sbid 0x282d ref_map(0x37c000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e83af90a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83abd570 loaded (sbid 0x282e ref_map(0x37e000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x37e000~2000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83abd570 loaded (sbid 0x282e ref_map(0x37e000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e83e6fea0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83e6f420 loaded (sbid 0x282f ref_map(0x380000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x380000~2000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83e6f420 loaded (sbid 0x282f ref_map(0x380000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e83e6f650 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd2d20 loaded (sbid 0x2830 ref_map(0x382000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x382000~2000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd2d20 loaded (sbid 0x2830 ref_map(0x382000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e83dd2690 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83dd2460 loaded (sbid 0x2831 ref_map(0x384000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x384000~2000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83dd2460 loaded (sbid 0x2831 ref_map(0x384000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558e83edefc0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8423ccb0 loaded (sbid 0x2832 ref_map(0x386000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x386000~2000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e8423ccb0 loaded (sbid 0x2832 ref_map(0x386000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558e83d02540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d029a0 loaded (sbid 0x2833 ref_map(0x388000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x388000~2000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83d029a0 loaded (sbid 0x2833 ref_map(0x388000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558e841dbc00 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e841dbb90 loaded (sbid 0x2834 ref_map(0x38a000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x38a000~2000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e841dbb90 loaded (sbid 0x2834 ref_map(0x38a000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558e841dbb20 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e841dbab0 loaded (sbid 0x2835 ref_map(0x38c000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x38c000~2000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e841dbab0 loaded (sbid 0x2835 ref_map(0x38c000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x16000~2000: 0x0~2000 Blob(0x558e841dba40 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e841db9d0 loaded (sbid 0x2836 ref_map(0x38e000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x38e000~2000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e841db9d0 loaded (sbid 0x2836 ref_map(0x38e000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x18000~2000: 0x0~2000 Blob(0x558e841db960 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e841db8f0 loaded (sbid 0x2837 ref_map(0x390000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x390000~2000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e841db8f0 loaded (sbid 0x2837 ref_map(0x390000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1a000~2000: 0x0~2000 Blob(0x558e841db880 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e817e9570 loaded (sbid 0x2838 ref_map(0x392000~2000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x392000~2000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e817e9570 loaded (sbid 0x2838 ref_map(0x392000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x1c000~1000: 0x0~1000 Blob(0x558e817e9730 blob([!~1000] csum+shared crc32c/0x1000) use_tracker(0x1000 0x0) SharedBlob(0x558e83d37f80 loaded (sbid 0x2839 ref_map(0x394000~1000=2)))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0x394000~1000] 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83d37f80 loaded (sbid 0x2839 ref_map(0x394000~1000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove gen and maybe_unshared_blobs 0x558e817e9570,0x558e83abd570,0x558e83d029a0,0x558e83d0f180,0x558e83d0f8f0,0x558e83d37f80,0x558e83dd2460,0x558e83dd2d20,0x558e83e6f420,0x558e841a28c0,0x558e841db8f0,0x558e841db9d0,0x558e841dbab0,0x558e841dbb90,0x558e8423ccb0 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove checking for unshareable blobs on 0x558e84641400 5#1:c8ce3553:::smithi13140796-29:head# 2019-12-07T19:34:14.633+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.3s5_head 5#1:c8ce3553:::smithi13140796-29:head#d = 0 2019-12-07T19:34:14.633+0000 7fe2c6263700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'._info' <- _info 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000461'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.633+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.3s5_head 5#1:c0000000::::head# = 0 2019-12-07T19:34:14.633+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e73800 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e73800 onodes shared_blobs 0x558e817e9570,0x558e83abd570,0x558e83d029a0,0x558e83d0f180,0x558e83d0f8f0,0x558e83d37f80,0x558e83dd2460,0x558e83dd2d20,0x558e83e6f420,0x558e841a28c0,0x558e841db8f0,0x558e841db9d0,0x558e841dbab0,0x558e841dbb90,0x558e8423ccb0 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2838 is 11 SharedBlob(0x558e817e9570 loaded (sbid 0x2838 ref_map(0x392000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x282e is 11 SharedBlob(0x558e83abd570 loaded (sbid 0x282e ref_map(0x37e000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2833 is 11 SharedBlob(0x558e83d029a0 loaded (sbid 0x2833 ref_map(0x388000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x282d is 11 SharedBlob(0x558e83d0f180 loaded (sbid 0x282d ref_map(0x37c000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x282c is 11 SharedBlob(0x558e83d0f8f0 loaded (sbid 0x282c ref_map(0x37a000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2839 is 11 SharedBlob(0x558e83d37f80 loaded (sbid 0x2839 ref_map(0x394000~1000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2831 is 11 SharedBlob(0x558e83dd2460 loaded (sbid 0x2831 ref_map(0x384000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2830 is 11 SharedBlob(0x558e83dd2d20 loaded (sbid 0x2830 ref_map(0x382000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x282f is 11 SharedBlob(0x558e83e6f420 loaded (sbid 0x282f ref_map(0x380000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x282b is 11 SharedBlob(0x558e841a28c0 loaded (sbid 0x282b ref_map(0x378000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2837 is 11 SharedBlob(0x558e841db8f0 loaded (sbid 0x2837 ref_map(0x390000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2836 is 11 SharedBlob(0x558e841db9d0 loaded (sbid 0x2836 ref_map(0x38e000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2835 is 11 SharedBlob(0x558e841dbab0 loaded (sbid 0x2835 ref_map(0x38c000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2834 is 11 SharedBlob(0x558e841dbb90 loaded (sbid 0x2834 ref_map(0x38a000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2832 is 11 SharedBlob(0x558e8423ccb0 loaded (sbid 0x2832 ref_map(0x386000~2000=1))) 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e73800 allocated 0x[] released 0x[] 2019-12-07T19:34:14.633+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73800 prepare 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e73800 2019-12-07T19:34:14.633+0000 7fe2c6263700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73800 io_done 2019-12-07T19:34:14.633+0000 7fe2c6263700 10 osd.0 37 dequeue_op 0x558e83dd9080 finish 2019-12-07T19:34:14.633+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.634+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:14.634+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.634+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1581 < min_flush_size 524288 2019-12-07T19:34:14.634+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8402ec80 had 1 2019-12-07T19:34:14.634+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84242c80 had 1 2019-12-07T19:34:14.634+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1604 < min_flush_size 524288 2019-12-07T19:34:14.634+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1604 < min_flush_size 524288 2019-12-07T19:34:14.634+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.634+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x16955c~644 to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.634+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x16955c 0x16955c~644 to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.634+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.634+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~140000 x_off 0x13955c 2019-12-07T19:34:14.634+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0x55c 2019-12-07T19:34:14.634+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.634+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xba0 and padding block with 0x460 2019-12-07T19:34:14.634+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x169ba0 2019-12-07T19:34:14.634+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.634+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:14.634+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:14.634+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:14.634+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000155266s (0.000000122s flush + 0.000155144s kv commit) 2019-12-07T19:34:14.634+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:14.634+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:14.634+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e73800> 2019-12-07T19:34:14.634+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.634+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73800 kv_submitted 2019-12-07T19:34:14.634+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e73800 2019-12-07T19:34:14.634+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73800 finishing 2019-12-07T19:34:14.634+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e73800 onodes 2019-12-07T19:34:14.634+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e73800 done 2019-12-07T19:34:14.634+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e83c6f2c0 q now empty 2019-12-07T19:34:14.634+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e73800 [] 2019-12-07T19:34:14.634+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:14.634+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:14.634+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- MOSDECSubOpWriteReply(1.3s0 37/28 ECSubWriteReply(tid=61, last_complete=37'13, committed=1, applied=1)) v2 -- 0x558e810fc400 con 0x558e818fe400 2019-12-07T19:34:14.634+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.634+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.634+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 703682 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.642+0000 7fe2d4a80700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 44 ==== osd_map(34..37 src has 1..37) v4 ==== 2496+0+0 (unknown 0 0 0) 0x558e846e2240 con 0x558e818fe000 2019-12-07T19:34:14.643+0000 7fe2d4a80700 20 osd.0 37 OSD::ms_dispatch: osd_map(34..37 src has 1..37) v4 2019-12-07T19:34:14.643+0000 7fe2d4a80700 10 osd.0 37 do_waiters -- start 2019-12-07T19:34:14.643+0000 7fe2d4a80700 10 osd.0 37 do_waiters -- finish 2019-12-07T19:34:14.643+0000 7fe2d4a80700 20 osd.0 37 _dispatch 0x558e846e2240 osd_map(34..37 src has 1..37) v4 2019-12-07T19:34:14.643+0000 7fe2d4a80700 3 osd.0 37 handle_osd_map epochs [34,37], i have 37, src has [1,37] 2019-12-07T19:34:14.643+0000 7fe2d4a80700 10 osd.0 37 no new maps here, dropping 2019-12-07T19:34:14.643+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 45 ==== MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=19, reqid=client.4540.0:305, at_version=37'4, trim_to=0'0, roll_forward_to=21'2)) v2 ==== 81183+0+0 (unknown 0 0 0) 0x558e819cf000 con 0x558e818fe000 2019-12-07T19:34:14.643+0000 7fe2e753b700 15 osd.0 37 enqueue_op 0x558e83edc580 prio 127 cost 0 latency 0.000041 epoch 37 MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=19, reqid=client.4540.0:305, at_version=37'4, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:34:14.643+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=19, reqid=client.4540.0:305, at_version=37'4, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=19, reqid=client.4540.0:305, at_version=37'4, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=19, reqid=client.4540.0:305, at_version=37'4, trim_to=0'0, roll_forward_to=21'2)) v2) prio 127 cost 0 e37) pg 0x558e81938000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 osd.0 37 dequeue_op 0x558e83edc580 prio 127 cost 0 latency 0.000093 MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=19, reqid=client.4540.0:305, at_version=37'4, trim_to=0'0, roll_forward_to=21'2)) v2 pg pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 osd.0 37 maybe_share_map con 0x558e818fe000 v2:172.21.15.109:6826/36939 map epoch 33 -> 37 (as per caller) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=1 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=19, reqid=client.4540.0:305, at_version=37'4, trim_to=0'0, roll_forward_to=21'2)) v2 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 snap_mapper.add_oid 1:276d5164:::smithi13140796-22:e c,d,e 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.onode(0x558e8125c280).flush flush done 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) omap_get_values meta oid #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 snap_mapper.get_snaps 1:276d5164:::smithi13140796-22:e got.empty() 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 snap_mapper.set_snaps 1:276d5164:::smithi13140796-22:e c,d,e 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 snap_mapper.set_snaps set OBJ_.1_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 snap_mapper.add_oid set SNA_1_000000000000000C_.1_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 snap_mapper.add_oid set SNA_1_000000000000000D_.1_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 snap_mapper.add_oid set SNA_1_000000000000000E_.1_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 21'2 (0'0,21'2] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] append_log log((0'0,21'2], crt=21'2) [37'3 (21'2) clone 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0,37'4 (21'2) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:305 0.000000 0 ObjectCleanRegions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0] 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 37'3 (0'0,37'3] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={} ps=[a~1]] add_log_entry 37'3 (21'2) clone 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={} ps=[a~1]] add_log_entry 37'4 (21'2) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:305 0.000000 0 ObjectCleanRegions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={} ps=[a~1]] append_log approx pg log length = 4 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 37'3, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e83e73200 seq 16 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) get_onode oid 1#1:276d5164:::smithi13140796-22:e# key 0x818000000000000001276d51'd!smithi13140796-22!='0x000000000000000effffffffffffffff'o' 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) r -2 v.len 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _clone 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# -> 1#1:276d5164:::smithi13140796-22:e# 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1171 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.onode(0x558e83b08c80).flush flush done 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.4s1_head 1#1:276d5164:::smithi13140796-22:e# 0x0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_clone_range 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# -> 1#1:276d5164:::smithi13140796-22:e# 0x0~16000 -> 0x0~16000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup src 0x0~2000: 0x0~2000 Blob(0x558e817e9f10 blob([0xb2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8000 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10298 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared Blob(0x558e817e9f10 blob([0xb2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8000 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared now Blob(0x558e817e9f10 blob([0xb2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8000 loaded (sbid 0x283a ref_map(0xb2000~2000=1)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup new Blob(0x558e8424b110 blob([0xb2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8000 loaded (sbid 0x283a ref_map(0xb2000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b110) get_ref 0x0~2000 Blob(0x558e8424b110 blob([0xb2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83af8000 loaded (sbid 0x283a ref_map(0xb2000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b110) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup dst 0x0~2000: 0x0~2000 Blob(0x558e8424b110 blob([0xb2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8000 loaded (sbid 0x283a ref_map(0xb2000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup src 0x2000~2000: 0x0~2000 Blob(0x558e817e9ea0 blob([0xb4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194efc0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10299 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared Blob(0x558e817e9ea0 blob([0xb4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194efc0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared now Blob(0x558e817e9ea0 blob([0xb4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194efc0 loaded (sbid 0x283b ref_map(0xb4000~2000=1)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup new Blob(0x558e8424b180 blob([0xb4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194efc0 loaded (sbid 0x283b ref_map(0xb4000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b180) get_ref 0x0~2000 Blob(0x558e8424b180 blob([0xb4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194efc0 loaded (sbid 0x283b ref_map(0xb4000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b180) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup dst 0x2000~2000: 0x0~2000 Blob(0x558e8424b180 blob([0xb4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194efc0 loaded (sbid 0x283b ref_map(0xb4000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup src 0x4000~2000: 0x0~2000 Blob(0x558e8194f110 blob([0xb6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f1f0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10300 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared Blob(0x558e8194f110 blob([0xb6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f1f0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared now Blob(0x558e8194f110 blob([0xb6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f1f0 loaded (sbid 0x283c ref_map(0xb6000~2000=1)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup new Blob(0x558e83dd2bd0 blob([0xb6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f1f0 loaded (sbid 0x283c ref_map(0xb6000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83dd2bd0) get_ref 0x0~2000 Blob(0x558e83dd2bd0 blob([0xb6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8194f1f0 loaded (sbid 0x283c ref_map(0xb6000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83dd2bd0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup dst 0x4000~2000: 0x0~2000 Blob(0x558e83dd2bd0 blob([0xb6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f1f0 loaded (sbid 0x283c ref_map(0xb6000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup src 0x6000~2000: 0x0~2000 Blob(0x558e83d36000 blob([0xb8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36070 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10301 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared Blob(0x558e83d36000 blob([0xb8000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36070 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared now Blob(0x558e83d36000 blob([0xb8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36070 loaded (sbid 0x283d ref_map(0xb8000~2000=1)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup new Blob(0x558e8424b1f0 blob([0xb8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36070 loaded (sbid 0x283d ref_map(0xb8000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b1f0) get_ref 0x0~2000 Blob(0x558e8424b1f0 blob([0xb8000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36070 loaded (sbid 0x283d ref_map(0xb8000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b1f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup dst 0x6000~2000: 0x0~2000 Blob(0x558e8424b1f0 blob([0xb8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36070 loaded (sbid 0x283d ref_map(0xb8000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup src 0x8000~2000: 0x0~2000 Blob(0x558e83d360e0 blob([0xba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36150 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10302 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared Blob(0x558e83d360e0 blob([0xba000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36150 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared now Blob(0x558e83d360e0 blob([0xba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36150 loaded (sbid 0x283e ref_map(0xba000~2000=1)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup new Blob(0x558e8424b0a0 blob([0xba000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36150 loaded (sbid 0x283e ref_map(0xba000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b0a0) get_ref 0x0~2000 Blob(0x558e8424b0a0 blob([0xba000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36150 loaded (sbid 0x283e ref_map(0xba000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b0a0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup dst 0x8000~2000: 0x0~2000 Blob(0x558e8424b0a0 blob([0xba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36150 loaded (sbid 0x283e ref_map(0xba000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup src 0xa000~2000: 0x0~2000 Blob(0x558e83d361c0 blob([0xbc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36230 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10303 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared Blob(0x558e83d361c0 blob([0xbc000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36230 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared now Blob(0x558e83d361c0 blob([0xbc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36230 loaded (sbid 0x283f ref_map(0xbc000~2000=1)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup new Blob(0x558e8424b030 blob([0xbc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36230 loaded (sbid 0x283f ref_map(0xbc000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b030) get_ref 0x0~2000 Blob(0x558e8424b030 blob([0xbc000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36230 loaded (sbid 0x283f ref_map(0xbc000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b030) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup dst 0xa000~2000: 0x0~2000 Blob(0x558e8424b030 blob([0xbc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36230 loaded (sbid 0x283f ref_map(0xbc000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup src 0xc000~2000: 0x0~2000 Blob(0x558e83d362a0 blob([0xbe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36310 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10304 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared Blob(0x558e83d362a0 blob([0xbe000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36310 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared now Blob(0x558e83d362a0 blob([0xbe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36310 loaded (sbid 0x2840 ref_map(0xbe000~2000=1)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup new Blob(0x558e8424af50 blob([0xbe000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36310 loaded (sbid 0x2840 ref_map(0xbe000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424af50) get_ref 0x0~2000 Blob(0x558e8424af50 blob([0xbe000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36310 loaded (sbid 0x2840 ref_map(0xbe000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424af50) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup dst 0xc000~2000: 0x0~2000 Blob(0x558e8424af50 blob([0xbe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36310 loaded (sbid 0x2840 ref_map(0xbe000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup src 0xe000~2000: 0x0~2000 Blob(0x558e83d36380 blob([0xc0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d363f0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10305 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared Blob(0x558e83d36380 blob([0xc0000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d363f0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared now Blob(0x558e83d36380 blob([0xc0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d363f0 loaded (sbid 0x2841 ref_map(0xc0000~2000=1)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup new Blob(0x558e8424aee0 blob([0xc0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d363f0 loaded (sbid 0x2841 ref_map(0xc0000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424aee0) get_ref 0x0~2000 Blob(0x558e8424aee0 blob([0xc0000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d363f0 loaded (sbid 0x2841 ref_map(0xc0000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424aee0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup dst 0xe000~2000: 0x0~2000 Blob(0x558e8424aee0 blob([0xc0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d363f0 loaded (sbid 0x2841 ref_map(0xc0000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup src 0x10000~2000: 0x0~2000 Blob(0x558e83d36460 blob([0xc2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d364d0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10306 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared Blob(0x558e83d36460 blob([0xc2000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d364d0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared now Blob(0x558e83d36460 blob([0xc2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d364d0 loaded (sbid 0x2842 ref_map(0xc2000~2000=1)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup new Blob(0x558e8424ae70 blob([0xc2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d364d0 loaded (sbid 0x2842 ref_map(0xc2000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424ae70) get_ref 0x0~2000 Blob(0x558e8424ae70 blob([0xc2000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d364d0 loaded (sbid 0x2842 ref_map(0xc2000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424ae70) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup dst 0x10000~2000: 0x0~2000 Blob(0x558e8424ae70 blob([0xc2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d364d0 loaded (sbid 0x2842 ref_map(0xc2000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup src 0x12000~2000: 0x0~2000 Blob(0x558e83d36540 blob([0xc4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d365b0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10307 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared Blob(0x558e83d36540 blob([0xc4000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d365b0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared now Blob(0x558e83d36540 blob([0xc4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d365b0 loaded (sbid 0x2843 ref_map(0xc4000~2000=1)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup new Blob(0x558e8424ae00 blob([0xc4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d365b0 loaded (sbid 0x2843 ref_map(0xc4000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424ae00) get_ref 0x0~2000 Blob(0x558e8424ae00 blob([0xc4000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d365b0 loaded (sbid 0x2843 ref_map(0xc4000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424ae00) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup dst 0x12000~2000: 0x0~2000 Blob(0x558e8424ae00 blob([0xc4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d365b0 loaded (sbid 0x2843 ref_map(0xc4000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup src 0x14000~2000: 0x0~2000 Blob(0x558e83d36620 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36690 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_blobid 10308 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared Blob(0x558e83d36620 blob([0xc6000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36690 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) make_blob_shared now Blob(0x558e83d36620 blob([0xc6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36690 loaded (sbid 0x2844 ref_map(0xc6000~2000=1)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup new Blob(0x558e83d36d90 blob([0xc6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36690 loaded (sbid 0x2844 ref_map(0xc6000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d36d90) get_ref 0x0~2000 Blob(0x558e83d36d90 blob([0xc6000~2000] csum+shared crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e83d36690 loaded (sbid 0x2844 ref_map(0xc6000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e83d36d90) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dup dst 0x14000~2000: 0x0~2000 Blob(0x558e83d36d90 blob([0xc6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36690 loaded (sbid 0x2844 ref_map(0xc6000~2000=2)))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) dirty_range mark inline shard dirty 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e84314108) dirty_range mark inline shard dirty 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _clone 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# -> 1#1:276d5164:::smithi13140796-22:e# = 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rmattr 1.4s1_head 1#1:276d5164:::smithi13140796-22:e# snapset 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rmattr 1.4s1_head 1#1:276d5164:::smithi13140796-22:e# snapset = 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.4s1_head 1#1:276d5164:::smithi13140796-22:e# 1 keys 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.4s1_head 1#1:276d5164:::smithi13140796-22:e# 1 keys = 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.4s1_head 1#1:276d5164:::smithi13140796-22:e# hinfo_key (42 bytes) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.4s1_head 1#1:276d5164:::smithi13140796-22:e# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) get_onode oid 1#1:276d5164:::smithi13140796-22:head#4 key 0x818000000000000001276d51'd!smithi13140796-22!='0xfffffffffffffffe0000000000000004'o' 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0).collection(1.4s1_head 0x558e819c54a0) r -2 v.len 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# -> 1#1:276d5164:::smithi13140796-22:head#4 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _rename 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# -> 1#1:276d5164:::smithi13140796-22:head#4 = 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _assign_nid 1172 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _touch 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# = 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 3 keys 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattrs 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 3 keys = 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# object_size 0 write_size 0 flags sequential_write+append_only 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _set_alloc_hint 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# object_size 0 write_size 0 flags sequential_write+append_only = 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 0x0~13000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write 1#1:276d5164:::smithi13140796-22:head# 0x0~13000 - have 0x0 (0) bytes fadvise_flags 0x0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _choose_write_options prefer csum_order 12 target_blob_size 0x2000 compress=1 buffered=0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_write_big 0x0~13000 target_blob_size 0x2000 compress 1 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write txc 0x558e83e73200 10 blobs 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write 0x2000 compressed to 0x2005 -> 0x3000 with 1, which is more than required 0x1c00 -> 0x2000, leaving uncompressed 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 fbmap_alloc 0x558e80233300 allocate 0x13000/1000,13000,0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x542000~a000/1000,13000,0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 fbmap_alloc 0x558e80233300 allocate extent: 0x56b000~9000/1000,13000,0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write prealloc [0x542000~a000,0x56b000~9000] 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8424b2d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8424b340 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8424b2d0 blob([0x542000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b340 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b2d0) get_ref 0x0~2000 Blob(0x558e8424b2d0 blob([0x542000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b340 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b2d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x0~2000: 0x0~2000 Blob(0x558e8424b2d0 blob([0x542000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8424b340 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e8424b358 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8424b3b0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8424b420 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8424b3b0 blob([0x544000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b420 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b3b0) get_ref 0x0~2000 Blob(0x558e8424b3b0 blob([0x544000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b420 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b3b0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x2000~2000: 0x0~2000 Blob(0x558e8424b3b0 blob([0x544000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8424b420 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e8424b438 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8424b490 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8424b500 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8424b490 blob([0x546000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b500 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b490) get_ref 0x0~2000 Blob(0x558e8424b490 blob([0x546000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b500 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b490) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x4000~2000: 0x0~2000 Blob(0x558e8424b490 blob([0x546000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8424b500 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e8424b518 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8424b570 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8424b5e0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8424b570 blob([0x548000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b5e0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b570) get_ref 0x0~2000 Blob(0x558e8424b570 blob([0x548000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b5e0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b570) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x6000~2000: 0x0~2000 Blob(0x558e8424b570 blob([0x548000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8424b5e0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e8424b5f8 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8424b650 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8424b6c0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8424b650 blob([0x54a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b6c0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b650) get_ref 0x0~2000 Blob(0x558e8424b650 blob([0x54a000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b6c0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b650) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x8000~2000: 0x0~2000 Blob(0x558e8424b650 blob([0x54a000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8424b6c0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e8424b6d8 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8424b730 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8424b7a0 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8424b730 blob([0x56b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b7a0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b730) get_ref 0x0~2000 Blob(0x558e8424b730 blob([0x56b000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b7a0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b730) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xa000~2000: 0x0~2000 Blob(0x558e8424b730 blob([0x56b000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8424b7a0 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e8424b7b8 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8424b810 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8424b880 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8424b810 blob([0x56d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b880 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b810) get_ref 0x0~2000 Blob(0x558e8424b810 blob([0x56d000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b880 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b810) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xc000~2000: 0x0~2000 Blob(0x558e8424b810 blob([0x56d000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8424b880 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e8424b898 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8424b8f0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8424b960 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8424b8f0 blob([0x56f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b960 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b8f0) get_ref 0x0~2000 Blob(0x558e8424b8f0 blob([0x56f000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424b960 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b8f0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0xe000~2000: 0x0~2000 Blob(0x558e8424b8f0 blob([0x56f000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8424b960 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e8424b978 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8424b9d0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8424ba40 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8424b9d0 blob([0x571000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424ba40 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b9d0) get_ref 0x0~2000 Blob(0x558e8424b9d0 blob([0x571000~2000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424ba40 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424b9d0) get_ref init 0x2000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x10000~2000: 0x0~2000 Blob(0x558e8424b9d0 blob([0x571000~2000] csum crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8424ba40 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e8424ba58 in 0x558e80219260) _discard 0x0~2000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write initialize csum setting for new blob Blob(0x558e8424bab0 blob([]) use_tracker(0x0 0x0) SharedBlob(0x558e8424bb20 sbid 0x0)) csum_type crc32c csum_order 12 csum_length 0x1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write blob Blob(0x558e8424bab0 blob([0x573000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424bb20 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424bab0) get_ref 0x0~1000 Blob(0x558e8424bab0 blob([0x573000~1000] csum crc32c/0x1000) use_tracker(0x0 0x0) SharedBlob(0x558e8424bb20 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.blob(0x558e8424bab0) get_ref init 0x1000, 1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_alloc_write lex 0x12000~1000: 0x0~1000 Blob(0x558e8424bab0 blob([0x573000~1000] csum crc32c/0x1000) use_tracker(0x1000 0x1000) SharedBlob(0x558e8424bb20 sbid 0x0)) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.BufferSpace(0x558e8424bb38 in 0x558e80219260) _discard 0x0~1000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write extending size to 0x13000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_write perform garbage collection for compressed extents, expected benefit = 0 AUs 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83f88108) dirty_range mark inline shard dirty 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _write 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# 0x0~13000 = 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _setattr 1.4s1_head 1#1:276d5164:::smithi13140796-22:head# hinfo_key (42 bytes) = 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.OBJ_.1_0000000000000001.4E6BA862.e.smithi13140796-22..' <- OBJ_.1_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000C_.1_0000000000000001.4E6BA862.e.smithi13140796-22..' <- SNA_1_000000000000000C_.1_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000D_.1_0000000000000001.4E6BA862.e.smithi13140796-22..' <- SNA_1_000000000000000D_.1_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x00000000000000000000000000000402'.SNA_1_000000000000000E_.1_0000000000000001.4E6BA862.e.smithi13140796-22..' <- SNA_1_000000000000000E_.1_0000000000000001.4E6BA862.e.smithi13140796-22.. 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys meta #-1:c0371625:::snapmapper:0# = 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.0000000037.00000000000000000003' <- 0000000037.00000000000000000003 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.0000000037.00000000000000000004' <- 0000000037.00000000000000000004 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._epoch' <- _epoch 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e83e73200 cost 126546 (11 ios * 4000 + 82546 bytes) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e83e73200 onodes 0x558e83b08c80,0x558e83f88000,0x558e84314000 shared_blobs 0x558e8194efc0,0x558e8194f1f0,0x558e83af8000,0x558e83d36070,0x558e83d36150,0x558e83d36230,0x558e83d36310,0x558e83d363f0,0x558e83d364d0,0x558e83d365b0,0x558e83d36690 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) update 1#1:276d5164:::smithi13140796-22:head#4 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83b08d88) update inline shard 300 bytes from 11 extents 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:276d5164:::smithi13140796-22:head#4 is 791 (485 bytes onode + 2 bytes spanning blobs + 304 bytes inline extents) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83f88108) update 1#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e83f88108) update inline shard 190 bytes from 10 extents 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:276d5164:::smithi13140796-22:head# is 753 (557 bytes onode + 2 bytes spanning blobs + 194 bytes inline extents) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e84314108) update 1#1:276d5164:::smithi13140796-22:e# 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore.extentmap(0x558e84314108) update inline shard 300 bytes from 11 extents 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _record_onode onode 1#1:276d5164:::smithi13140796-22:e# is 718 (412 bytes onode + 2 bytes spanning blobs + 304 bytes inline extents) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x283b is 11 SharedBlob(0x558e8194efc0 loaded (sbid 0x283b ref_map(0xb4000~2000=2))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x283c is 11 SharedBlob(0x558e8194f1f0 loaded (sbid 0x283c ref_map(0xb6000~2000=2))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x283a is 11 SharedBlob(0x558e83af8000 loaded (sbid 0x283a ref_map(0xb2000~2000=2))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x283d is 11 SharedBlob(0x558e83d36070 loaded (sbid 0x283d ref_map(0xb8000~2000=2))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x283e is 11 SharedBlob(0x558e83d36150 loaded (sbid 0x283e ref_map(0xba000~2000=2))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x283f is 11 SharedBlob(0x558e83d36230 loaded (sbid 0x283f ref_map(0xbc000~2000=2))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2840 is 11 SharedBlob(0x558e83d36310 loaded (sbid 0x2840 ref_map(0xbe000~2000=2))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2841 is 11 SharedBlob(0x558e83d363f0 loaded (sbid 0x2841 ref_map(0xc0000~2000=2))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2842 is 11 SharedBlob(0x558e83d364d0 loaded (sbid 0x2842 ref_map(0xc2000~2000=2))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2843 is 11 SharedBlob(0x558e83d365b0 loaded (sbid 0x2843 ref_map(0xc4000~2000=2))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2844 is 11 SharedBlob(0x558e83d36690 loaded (sbid 0x2844 ref_map(0xc6000~2000=2))) 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e83e73200 allocated 0x[542000~a000,56b000~9000] released 0x[] 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 freelist allocate 0x542000~a000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 freelist _xor first_key 0x500000 last_key 0x500000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 freelist allocate 0x56b000~9000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 20 freelist _xor first_key 0x500000 last_key 0x500000 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73200 prepare 2019-12-07T19:34:14.643+0000 7fe2c9a6a700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_aio_submit txc 0x558e83e73200 2019-12-07T19:34:14.644+0000 7fe2c9a6a700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 lua=21'2 crt=21'2 lcod 21'2 active mbc={} ps=[a~1]] op_applied version 37'4 2019-12-07T19:34:14.644+0000 7fe2c9a6a700 10 osd.0 37 dequeue_op 0x558e83edc580 finish 2019-12-07T19:34:14.644+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.644+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.644+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73200 aio_wait 2019-12-07T19:34:14.644+0000 7fe2e049d700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e83e73200 2019-12-07T19:34:14.644+0000 7fe2e049d700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73200 io_done 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread num_aios=1 force_flush=1, flushing, deferred done->stable 2019-12-07T19:34:14.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5373 < min_flush_size 524288 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b08c80 had 1 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83f88000 had 1 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e84314000 had 1 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e8125c280 had 1 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:34:14.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5396 < min_flush_size 524288 2019-12-07T19:34:14.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 5396 < min_flush_size 524288 2019-12-07T19:34:14.644+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.644+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x169ba0~1514 to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.644+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x169ba0 0x169ba0~1514 to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~140000 x_off 0x139ba0 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xba0 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0xb4 and padding block with 0xf4c 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x16b0b4 2019-12-07T19:34:14.644+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.644+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:14.644+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000161626s (0.000005029s flush + 0.000156597s kv commit) 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:14.644+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e83e73200> 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.644+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73200 kv_submitted 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e83e73200 2019-12-07T19:34:14.644+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e83e73200 finishing 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e83e73200 onodes 0x558e83b08c80,0x558e83f88000,0x558e84314000 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8424b358 in 0x558e80219260) _finish_write discard buffer(0x558e83c8e750 space 0x558e8424b358 0x0~2000 writing nocache) 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8424b438 in 0x558e80219260) _finish_write discard buffer(0x558e83c8fef0 space 0x558e8424b438 0x0~2000 writing nocache) 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8424b518 in 0x558e80219260) _finish_write discard buffer(0x558e84246c60 space 0x558e8424b518 0x0~2000 writing nocache) 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8424b5f8 in 0x558e80219260) _finish_write discard buffer(0x558e84246cf0 space 0x558e8424b5f8 0x0~2000 writing nocache) 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8424b6d8 in 0x558e80219260) _finish_write discard buffer(0x558e84246d80 space 0x558e8424b6d8 0x0~2000 writing nocache) 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8424b7b8 in 0x558e80219260) _finish_write discard buffer(0x558e84246e10 space 0x558e8424b7b8 0x0~2000 writing nocache) 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8424b898 in 0x558e80219260) _finish_write discard buffer(0x558e83d11290 space 0x558e8424b898 0x0~2000 writing nocache) 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8424b978 in 0x558e80219260) _finish_write discard buffer(0x558e83c8fd40 space 0x558e8424b978 0x0~2000 writing nocache) 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8424ba58 in 0x558e80219260) _finish_write discard buffer(0x558e84246ea0 space 0x558e8424ba58 0x0~2000 writing nocache) 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore.BufferSpace(0x558e8424bb38 in 0x558e80219260) _finish_write discard buffer(0x558e84246f30 space 0x558e8424bb38 0x0~1000 writing nocache) 2019-12-07T19:34:14.644+0000 7fe2c9a6a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=19, last_complete=37'4, committed=1, applied=1)) v2 -- 0x558e818eec00 con 0x558e818fe000 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e83e73200 done 2019-12-07T19:34:14.644+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df900 q now empty 2019-12-07T19:34:14.644+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e83e73200 [] 2019-12-07T19:34:14.644+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:14.644+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:14.644+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.647+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 46 ==== MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=21, reqid=client.4540.0:305, at_version=0'0, trim_to=0'0, roll_forward_to=37'4)) v2 ==== 910+0+0 (unknown 0 0 0) 0x558e819d1800 con 0x558e818fe000 2019-12-07T19:34:14.647+0000 7fe2e753b700 15 osd.0 37 enqueue_op 0x558e83edc000 prio 127 cost 0 latency 0.000031 epoch 37 MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=21, reqid=client.4540.0:305, at_version=0'0, trim_to=0'0, roll_forward_to=37'4)) v2 2019-12-07T19:34:14.647+0000 7fe2e753b700 20 osd.0 op_wq(4) _enqueue OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=21, reqid=client.4540.0:305, at_version=0'0, trim_to=0'0, roll_forward_to=37'4)) v2) prio 127 cost 0 e37) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=21, reqid=client.4540.0:305, at_version=0'0, trim_to=0'0, roll_forward_to=37'4)) v2) prio 127 cost 0 e37) queued 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process 1.4s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process OpSchedulerItem(1.4s1 PGOpItem(op=MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=21, reqid=client.4540.0:305, at_version=0'0, trim_to=0'0, roll_forward_to=37'4)) v2) prio 127 cost 0 e37) pg 0x558e81938000 2019-12-07T19:34:14.648+0000 7fe2c5a62700 10 osd.0 37 dequeue_op 0x558e83edc000 prio 127 cost 0 latency 0.000488 MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=21, reqid=client.4540.0:305, at_version=0'0, trim_to=0'0, roll_forward_to=37'4)) v2 pg pg[1.4s1( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] 2019-12-07T19:34:14.648+0000 7fe2c5a62700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] _handle_message: MOSDECSubOpWrite(1.4s1 37/18 ECSubWrite(tid=21, reqid=client.4540.0:305, at_version=0'0, trim_to=0'0, roll_forward_to=37'4)) v2 2019-12-07T19:34:14.648+0000 7fe2c5a62700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] append_log log((0'0,37'4], crt=21'2) [] 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 osd.0 pg_epoch: 37 pg[1.4s1( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=37'4 active mbc={} ps=[a~1]] rollforward: entry=37'3 (21'2) clone 1:276d5164:::smithi13140796-22:e by unknown.0.0:0 2019-12-07T19:33:58.661803+0000 0 snaps [e,d,c] ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 osd.0 pg_epoch: 37 pg[1.4s1( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=37'4 active mbc={} ps=[a~1]] rollforward: entry=37'4 (21'2) modify 1:276d5164:::smithi13140796-22:head by client.4540.0:305 0.000000 0 ObjectCleanRegions clean_offsets: [360448~18446744073709191167], clean_omap: 1, new_object: 0 2019-12-07T19:34:14.648+0000 7fe2c5a62700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=37'4 active mbc={} ps=[a~1]] append_log approx pg log length = 4 2019-12-07T19:34:14.648+0000 7fe2c5a62700 10 osd.0 pg_epoch: 37 pg[1.4s1( v 37'4 (0'0,37'4] local-lis/les=18/19 n=2 ec=18/18 lis/c=18/18 les/c/f=19/19/0 sis=18) [7,0,1,5,2,4]p7(0) r=1 lpr=18 luod=0'0 crt=37'4 active mbc={} ps=[a~1]] append_log transaction_applied = 1 2019-12-07T19:34:14.648+0000 7fe2c5a62700 10 trim proposed trim_to = 0'0 2019-12-07T19:34:14.648+0000 7fe2c5a62700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2019-12-07T19:34:14.648+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) queue_transactions ch 0x558e819c54a0 1.4s1_head 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_create osr 0x558e811df900 = 0x558e811fac00 seq 17 2019-12-07T19:34:14.648+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.4s1_head 1#1:276d5164:::smithi13140796-22:head#4 onode 0x558e83b08c80 txc 0x558e811fac00 2019-12-07T19:34:14.648+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) _do_truncate 1.4s1_head 1#1:276d5164:::smithi13140796-22:head#4 0x0 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore.blob(0x558e817e9f10) put_ref 0x0~2000 Blob(0x558e817e9f10 blob([0xb2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83af8000 loaded (sbid 0x283a ref_map(0xb2000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore.blob(0x558e817e9ea0) put_ref 0x0~2000 Blob(0x558e817e9ea0 blob([0xb4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194efc0 loaded (sbid 0x283b ref_map(0xb4000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore.blob(0x558e8194f110) put_ref 0x0~2000 Blob(0x558e8194f110 blob([0xb6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e8194f1f0 loaded (sbid 0x283c ref_map(0xb6000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore.blob(0x558e83d36000) put_ref 0x0~2000 Blob(0x558e83d36000 blob([0xb8000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36070 loaded (sbid 0x283d ref_map(0xb8000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore.blob(0x558e83d360e0) put_ref 0x0~2000 Blob(0x558e83d360e0 blob([0xba000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36150 loaded (sbid 0x283e ref_map(0xba000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore.blob(0x558e83d361c0) put_ref 0x0~2000 Blob(0x558e83d361c0 blob([0xbc000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36230 loaded (sbid 0x283f ref_map(0xbc000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore.blob(0x558e83d362a0) put_ref 0x0~2000 Blob(0x558e83d362a0 blob([0xbe000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36310 loaded (sbid 0x2840 ref_map(0xbe000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore.blob(0x558e83d36380) put_ref 0x0~2000 Blob(0x558e83d36380 blob([0xc0000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d363f0 loaded (sbid 0x2841 ref_map(0xc0000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore.blob(0x558e83d36460) put_ref 0x0~2000 Blob(0x558e83d36460 blob([0xc2000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d364d0 loaded (sbid 0x2842 ref_map(0xc2000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore.blob(0x558e83d36540) put_ref 0x0~2000 Blob(0x558e83d36540 blob([0xc4000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d365b0 loaded (sbid 0x2843 ref_map(0xc4000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore.blob(0x558e83d36620) put_ref 0x0~2000 Blob(0x558e83d36620 blob([0xc6000~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[1000,1000]) SharedBlob(0x558e83d36690 loaded (sbid 0x2844 ref_map(0xc6000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore.extentmap(0x558e83b08d88) dirty_range mark inline shard dirty 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x0~2000: 0x0~2000 Blob(0x558e817e9f10 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83af8000 loaded (sbid 0x283a ref_map(0xb2000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xb2000~2000] 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83af8000 loaded (sbid 0x283a ref_map(0xb2000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x2000~2000: 0x0~2000 Blob(0x558e817e9ea0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8194efc0 loaded (sbid 0x283b ref_map(0xb4000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xb4000~2000] 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e8194efc0 loaded (sbid 0x283b ref_map(0xb4000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x4000~2000: 0x0~2000 Blob(0x558e8194f110 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e8194f1f0 loaded (sbid 0x283c ref_map(0xb6000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xb6000~2000] 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e8194f1f0 loaded (sbid 0x283c ref_map(0xb6000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x6000~2000: 0x0~2000 Blob(0x558e83d36000 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d36070 loaded (sbid 0x283d ref_map(0xb8000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xb8000~2000] 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83d36070 loaded (sbid 0x283d ref_map(0xb8000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x8000~2000: 0x0~2000 Blob(0x558e83d360e0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d36150 loaded (sbid 0x283e ref_map(0xba000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xba000~2000] 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83d36150 loaded (sbid 0x283e ref_map(0xba000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xa000~2000: 0x0~2000 Blob(0x558e83d361c0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d36230 loaded (sbid 0x283f ref_map(0xbc000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xbc000~2000] 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83d36230 loaded (sbid 0x283f ref_map(0xbc000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xc000~2000: 0x0~2000 Blob(0x558e83d362a0 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d36310 loaded (sbid 0x2840 ref_map(0xbe000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xbe000~2000] 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83d36310 loaded (sbid 0x2840 ref_map(0xbe000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0xe000~2000: 0x0~2000 Blob(0x558e83d36380 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d363f0 loaded (sbid 0x2841 ref_map(0xc0000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xc0000~2000] 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83d363f0 loaded (sbid 0x2841 ref_map(0xc0000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x10000~2000: 0x0~2000 Blob(0x558e83d36460 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d364d0 loaded (sbid 0x2842 ref_map(0xc2000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xc2000~2000] 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83d364d0 loaded (sbid 0x2842 ref_map(0xc2000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x12000~2000: 0x0~2000 Blob(0x558e83d36540 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d365b0 loaded (sbid 0x2843 ref_map(0xc4000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xc4000~2000] 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83d365b0 loaded (sbid 0x2843 ref_map(0xc4000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish lex_old 0x14000~2000: 0x0~2000 Blob(0x558e83d36620 blob([!~2000] csum+shared crc32c/0x1000) use_tracker(0x2*0x1000 0x[0,0]) SharedBlob(0x558e83d36690 loaded (sbid 0x2844 ref_map(0xc6000~2000=2)))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish blob release [0xc6000~2000] 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _wctx_finish shared_blob release [] from SharedBlob(0x558e83d36690 loaded (sbid 0x2844 ref_map(0xc6000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove gen and maybe_unshared_blobs 0x558e8194efc0,0x558e8194f1f0,0x558e83af8000,0x558e83d36070,0x558e83d36150,0x558e83d36230,0x558e83d36310,0x558e83d363f0,0x558e83d364d0,0x558e83d365b0,0x558e83d36690 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _do_remove checking for unshareable blobs on 0x558e83f88000 1#1:276d5164:::smithi13140796-22:head# 2019-12-07T19:34:14.648+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _remove 1.4s1_head 1#1:276d5164:::smithi13140796-22:head#4 = 0 2019-12-07T19:34:14.648+0000 7fe2c5a62700 15 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'._info' <- _info 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.can_rollback_to' <- can_rollback_to 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 0x0000000000000431'.rollback_info_trimmed_to' <- rollback_info_trimmed_to 2019-12-07T19:34:14.648+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _omap_setkeys 1.4s1_head 1#1:20000000::::head# = 0 2019-12-07T19:34:14.648+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_calc_cost 0x558e811fac00 cost 5445 (1 ios * 4000 + 1445 bytes) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes txc 0x558e811fac00 onodes shared_blobs 0x558e8194efc0,0x558e8194f1f0,0x558e83af8000,0x558e83d36070,0x558e83d36150,0x558e83d36230,0x558e83d36310,0x558e83d363f0,0x558e83d364d0,0x558e83d365b0,0x558e83d36690 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x283b is 11 SharedBlob(0x558e8194efc0 loaded (sbid 0x283b ref_map(0xb4000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x283c is 11 SharedBlob(0x558e8194f1f0 loaded (sbid 0x283c ref_map(0xb6000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x283a is 11 SharedBlob(0x558e83af8000 loaded (sbid 0x283a ref_map(0xb2000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x283d is 11 SharedBlob(0x558e83d36070 loaded (sbid 0x283d ref_map(0xb8000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x283e is 11 SharedBlob(0x558e83d36150 loaded (sbid 0x283e ref_map(0xba000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x283f is 11 SharedBlob(0x558e83d36230 loaded (sbid 0x283f ref_map(0xbc000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2840 is 11 SharedBlob(0x558e83d36310 loaded (sbid 0x2840 ref_map(0xbe000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2841 is 11 SharedBlob(0x558e83d363f0 loaded (sbid 0x2841 ref_map(0xc0000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2842 is 11 SharedBlob(0x558e83d364d0 loaded (sbid 0x2842 ref_map(0xc2000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2843 is 11 SharedBlob(0x558e83d365b0 loaded (sbid 0x2843 ref_map(0xc4000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_write_nodes shared_blob 0x2844 is 11 SharedBlob(0x558e83d36690 loaded (sbid 0x2844 ref_map(0xc6000~2000=1))) 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finalize_kv txc 0x558e811fac00 allocated 0x[] released 0x[] 2019-12-07T19:34:14.648+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 prepare 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish_io 0x558e811fac00 2019-12-07T19:34:14.648+0000 7fe2c5a62700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 io_done 2019-12-07T19:34:14.648+0000 7fe2c5a62700 10 osd.0 37 dequeue_op 0x558e83edc000 finish 2019-12-07T19:34:14.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread wake 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committing 1 submitting 1 deferred done 0 stable 0 2019-12-07T19:34:14.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1485 < min_flush_size 524288 2019-12-07T19:34:14.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e81a02c80 had 1 2019-12-07T19:34:14.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_apply_kv onode 0x558e83b08c80 had 1 2019-12-07T19:34:14.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1508 < min_flush_size 524288 2019-12-07T19:34:14.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 ignoring, length 1508 < min_flush_size 524288 2019-12-07T19:34:14.648+0000 7fe2d7a86700 10 bluefs _fsync 0x558e8107a280 file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.648+0000 7fe2d7a86700 10 bluefs _flush 0x558e8107a280 0x16b0b4~5e4 to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.648+0000 7fe2d7a86700 10 bluefs _flush_range 0x558e8107a280 pos 0x16b0b4 0x16b0b4~5e4 to file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.648+0000 7fe2d7a86700 20 bluefs _flush_range file now file(ino 16 size 0x170000 mtime 2019-12-07T19:34:13.614651+0000 allocated 170000 extents [1:0xacd120000~10000,1:0xacd0f0000~10000,1:0xacd130000~10000,1:0xacd160000~140000]) 2019-12-07T19:34:14.648+0000 7fe2d7a86700 20 bluefs _flush_range in 1:0xacd160000~140000 x_off 0x13b0b4 2019-12-07T19:34:14.648+0000 7fe2d7a86700 20 bluefs _flush_range using partial tail 0xb4 2019-12-07T19:34:14.648+0000 7fe2d7a86700 20 bluefs _flush_range waiting for previous aio to complete 2019-12-07T19:34:14.648+0000 7fe2d7a86700 20 bluefs _flush_range caching tail of 0x698 and padding block with 0x968 2019-12-07T19:34:14.648+0000 7fe2d7a86700 20 bluefs _flush_range h 0x558e8107a280 pos now 0x16b698 2019-12-07T19:34:14.648+0000 7fe2d7a86700 10 bluefs _claim_completed_aios got 0 aios 2019-12-07T19:34:14.648+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 2019-12-07T19:34:14.648+0000 7fe2d7a86700 10 bluefs wait_for_aio 0x558e8107a280 done in 0.000000 2019-12-07T19:34:14.648+0000 7fe2d7a86700 20 bluefs flush_bdev 2019-12-07T19:34:14.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread committed 1 cleaned 0 in 0.000103402s (0.000000144s flush + 0.000103258s kv commit) 2019-12-07T19:34:14.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread wake 2019-12-07T19:34:14.648+0000 7fe2d7a86700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_sync_thread sleep 2019-12-07T19:34:14.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread kv_committed <0x558e811fac00> 2019-12-07T19:34:14.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread deferred_stable <> 2019-12-07T19:34:14.648+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 kv_submitted 2019-12-07T19:34:14.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_committed_kv txc 0x558e811fac00 2019-12-07T19:34:14.648+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_state_proc txc 0x558e811fac00 finishing 2019-12-07T19:34:14.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish 0x558e811fac00 onodes 2019-12-07T19:34:14.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish txc 0x558e811fac00 done 2019-12-07T19:34:14.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _txc_finish osr 0x558e811df900 q now empty 2019-12-07T19:34:14.648+0000 7fe2dca90700 10 bluestore(/var/lib/ceph/osd/ceph-0) _txc_release_alloc(sync) 0x558e811fac00 [] 2019-12-07T19:34:14.648+0000 7fe2dca90700 10 fbmap_alloc 0x558e80233300 release done 2019-12-07T19:34:14.648+0000 7fe2c5a62700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.648+0000 7fe2dca90700 20 bluestore(/var/lib/ceph/osd/ceph-0) _kv_finalize_thread sleep 2019-12-07T19:34:14.648+0000 7fe2c9a6a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- MOSDECSubOpWriteReply(1.4s0 37/18 ECSubWriteReply(tid=21, last_complete=37'4, committed=1, applied=1)) v2 -- 0x558e818efa00 con 0x558e818fe000 2019-12-07T19:34:14.648+0000 7fe2c9a6a700 20 osd.0 op_wq(4) _process empty q, waiting 2019-12-07T19:34:14.655+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 140 ==== pg_lease(1.bs1 pg_lease(ru 46.104938842s ub 54.105088836s int 16s) e37/37) v1 ==== 58+0+0 (unknown 0 0 0) 0x558e810fc400 con 0x558e818fe400 2019-12-07T19:34:14.655+0000 7fe2e6d3a700 15 osd.0 37 enqueue_peering_evt 1.bs1 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.104938842s ub 54.105088836s int 16s) 2019-12-07T19:34:14.655+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.104938842s ub 54.105088836s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.655+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.655+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.104938842s ub 54.105088836s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.655+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process 1.bs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.655+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.bs1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.104938842s ub 54.105088836s int 16s)) prio 255 cost 10 e37) pg 0x558e83c78000 2019-12-07T19:34:14.655+0000 7fe2ca26b700 10 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.104938842s ub 54.105088836s int 16s) 2019-12-07T19:34:14.655+0000 7fe2ca26b700 10 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.104938842s ub 54.105088836s int 16s) 2019-12-07T19:34:14.655+0000 7fe2ca26b700 20 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] peer_clock_delta_ub 18446744073.704155227s -> ru 46.110335231s 2019-12-07T19:34:14.655+0000 7fe2ca26b700 20 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] proc_lease readable_until now 46.110335231s 2019-12-07T19:34:14.655+0000 7fe2ca26b700 20 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] peer_clock_delta_lb 18446744073.703802869s -> ruub 54.110837583s 2019-12-07T19:34:14.655+0000 7fe2ca26b700 20 osd.0 pg_epoch: 37 pg[1.bs1( v 21'12 (0'0,21'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,0,2,7,1,5]p4(0) r=1 lpr=28 luod=0'0 crt=21'12 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.110837583s 2019-12-07T19:34:14.655+0000 7fe2ca26b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease_ack(1.bs0 pg_lease_ack(ruub 54.105088836s) e37/37) v1 -- 0x558e84738f00 con 0x558e818fe400 2019-12-07T19:34:14.655+0000 7fe2ca26b700 10 log is not dirty 2019-12-07T19:34:14.655+0000 7fe2ca26b700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.656+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 141 ==== pg_lease(1.3s5 pg_lease(ru 46.105685108s ub 54.105801978s int 16s) e37/37) v1 ==== 58+0+0 (unknown 0 0 0) 0x558e810fc400 con 0x558e818fe400 2019-12-07T19:34:14.656+0000 7fe2e6d3a700 15 osd.0 37 enqueue_peering_evt 1.3s5 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.105685108s ub 54.105801978s int 16s) 2019-12-07T19:34:14.656+0000 7fe2e6d3a700 20 osd.0 op_wq(3) _enqueue OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.105685108s ub 54.105801978s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.656+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.105685108s ub 54.105801978s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.656+0000 7fe2c6263700 20 osd.0 op_wq(3) _process 1.3s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7fe2c6263700 20 osd.0 op_wq(3) _process OpSchedulerItem(1.3s5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.105685108s ub 54.105801978s int 16s)) prio 255 cost 10 e37) pg 0x558e81231000 2019-12-07T19:34:14.656+0000 7fe2c6263700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.4 pg_lease(ru 46.105685108s ub 54.105801978s int 16s) 2019-12-07T19:34:14.656+0000 7fe2c6263700 10 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.105685108s ub 54.105801978s int 16s) 2019-12-07T19:34:14.656+0000 7fe2c6263700 20 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] peer_clock_delta_ub 18446744073.704155227s -> ru 46.111081497s 2019-12-07T19:34:14.656+0000 7fe2c6263700 20 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] proc_lease readable_until now 46.111081497s 2019-12-07T19:34:14.656+0000 7fe2c6263700 20 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] peer_clock_delta_lb 18446744073.703802869s -> ruub 54.111550725s 2019-12-07T19:34:14.656+0000 7fe2c6263700 20 osd.0 pg_epoch: 37 pg[1.3s5( v 37'13 (0'0,37'13] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [4,1,2,5,7,0]p4(0) r=5 lpr=28 luod=0'0 crt=37'13 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.111550725s 2019-12-07T19:34:14.656+0000 7fe2c6263700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease_ack(1.3s0 pg_lease_ack(ruub 54.105801978s) e37/37) v1 -- 0x558e8122de00 con 0x558e818fe400 2019-12-07T19:34:14.656+0000 7fe2c6263700 10 log is not dirty 2019-12-07T19:34:14.656+0000 7fe2c6263700 20 osd.0 op_wq(3) _process empty q, waiting 2019-12-07T19:34:14.656+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 265 ==== pg_lease(1.8s5 pg_lease(ru 46.113761808s ub 54.113902230s int 16s) e37/37) v1 ==== 58+0+0 (unknown 0 0 0) 0x558e84052c00 con 0x558e818d7c00 2019-12-07T19:34:14.656+0000 7fe2e7d3c700 15 osd.0 37 enqueue_peering_evt 1.8s5 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.113761808s ub 54.113902230s int 16s) 2019-12-07T19:34:14.656+0000 7fe2e7d3c700 20 osd.0 op_wq(0) _enqueue OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.113761808s ub 54.113902230s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.656+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.113761808s ub 54.113902230s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.656+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process 1.8s5 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.656+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process OpSchedulerItem(1.8s5 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.113761808s ub 54.113902230s int 16s)) prio 255 cost 10 e37) pg 0x558e841a4000 2019-12-07T19:34:14.656+0000 7fe2c7a66700 10 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.113761808s ub 54.113902230s int 16s) 2019-12-07T19:34:14.656+0000 7fe2c7a66700 10 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.113761808s ub 54.113902230s int 16s) 2019-12-07T19:34:14.656+0000 7fe2c7a66700 20 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] peer_clock_delta_ub 0.002416414s -> ru 46.111345394s 2019-12-07T19:34:14.656+0000 7fe2c7a66700 20 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] proc_lease readable_until now 46.111345394s 2019-12-07T19:34:14.656+0000 7fe2c7a66700 20 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] peer_clock_delta_lb 0.002061166s -> ruub 54.111841064s 2019-12-07T19:34:14.656+0000 7fe2c7a66700 20 osd.0 pg_epoch: 37 pg[1.8s5( v 36'33 (0'0,36'33] local-lis/les=28/29 n=6 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,2,3,0]p1(0) r=5 lpr=28 luod=0'0 crt=36'33 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.111841064s 2019-12-07T19:34:14.656+0000 7fe2c7a66700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.8s0 pg_lease_ack(ruub 54.113902230s) e37/37) v1 -- 0x558e8403f860 con 0x558e818d7c00 2019-12-07T19:34:14.656+0000 7fe2c7a66700 10 log is not dirty 2019-12-07T19:34:14.656+0000 7fe2c7a66700 20 osd.0 op_wq(0) _process empty q, waiting 2019-12-07T19:34:14.656+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 266 ==== pg_lease(1.9s3 pg_lease(ru 46.114035089s ub 54.114343743s int 16s) e37/37) v1 ==== 58+0+0 (unknown 0 0 0) 0x558e84052c00 con 0x558e818d7c00 2019-12-07T19:34:14.656+0000 7fe2e7d3c700 15 osd.0 37 enqueue_peering_evt 1.9s3 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114035089s ub 54.114343743s int 16s) 2019-12-07T19:34:14.656+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114035089s ub 54.114343743s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.657+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.657+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114035089s ub 54.114343743s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.657+0000 7fe2c7265700 20 osd.0 op_wq(1) _process 1.9s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.657+0000 7fe2c7265700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.9s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114035089s ub 54.114343743s int 16s)) prio 255 cost 10 e37) pg 0x558e8192b000 2019-12-07T19:34:14.657+0000 7fe2c7265700 10 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114035089s ub 54.114343743s int 16s) 2019-12-07T19:34:14.657+0000 7fe2c7265700 10 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.114035089s ub 54.114343743s int 16s) 2019-12-07T19:34:14.657+0000 7fe2c7265700 20 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] peer_clock_delta_ub 0.002416414s -> ru 46.111618675s 2019-12-07T19:34:14.657+0000 7fe2c7265700 20 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] proc_lease readable_until now 46.111618675s 2019-12-07T19:34:14.657+0000 7fe2c7265700 20 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] peer_clock_delta_lb 0.002061166s -> ruub 54.112282577s 2019-12-07T19:34:14.657+0000 7fe2c7265700 20 osd.0 pg_epoch: 37 pg[1.9s3( v 34'17 (0'0,34'17] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,5,0,7,3]p1(0) r=3 lpr=28 luod=0'0 crt=34'17 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.112282577s 2019-12-07T19:34:14.657+0000 7fe2c7265700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.9s0 pg_lease_ack(ruub 54.114343743s) e37/37) v1 -- 0x558e845863c0 con 0x558e818d7c00 2019-12-07T19:34:14.657+0000 7fe2c7265700 10 log is not dirty 2019-12-07T19:34:14.657+0000 7fe2c7265700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:14.657+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 136 ==== pg_lease(1.fs1 pg_lease(ru 46.114604348s ub 54.115092785s int 16s) e37/37) v1 ==== 58+0+0 (unknown 0 0 0) 0x558e84052c00 con 0x558e818d7000 2019-12-07T19:34:14.657+0000 7fe2e7d3c700 15 osd.0 37 enqueue_peering_evt 1.fs1 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.114604348s ub 54.115092785s int 16s) 2019-12-07T19:34:14.657+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.114604348s ub 54.115092785s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.657+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 137 ==== pg_lease(1.1s1 pg_lease(ru 46.115024122s ub 54.115157360s int 16s) e37/37) v1 ==== 58+0+0 (unknown 0 0 0) 0x558e84052c00 con 0x558e818d7000 2019-12-07T19:34:14.657+0000 7fe2e7d3c700 15 osd.0 37 enqueue_peering_evt 1.1s1 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.115024122s ub 54.115157360s int 16s) 2019-12-07T19:34:14.657+0000 7fe2e7d3c700 20 osd.0 op_wq(1) _enqueue OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.115024122s ub 54.115157360s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.657+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.657+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.115024122s ub 54.115157360s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.657+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process 1.1s1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.657+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process OpSchedulerItem(1.1s1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.115024122s ub 54.115157360s int 16s)) prio 255 cost 10 e37) pg 0x558e819e4000 2019-12-07T19:34:14.657+0000 7fe2cb26d700 10 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.115024122s ub 54.115157360s int 16s) 2019-12-07T19:34:14.657+0000 7fe2cb26d700 10 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.115024122s ub 54.115157360s int 16s) 2019-12-07T19:34:14.657+0000 7fe2cb26d700 20 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] peer_clock_delta_ub 0.002985006s -> ru 46.112039116s 2019-12-07T19:34:14.657+0000 7fe2cb26d700 20 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] proc_lease readable_until now 46.112039116s 2019-12-07T19:34:14.657+0000 7fe2cb26d700 20 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] peer_clock_delta_lb 0.002678348s -> ruub 54.112479012s 2019-12-07T19:34:14.657+0000 7fe2cb26d700 20 osd.0 pg_epoch: 37 pg[1.1s1( v 29'9 (0'0,29'9] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,2,4,5,1]p3(0) r=1 lpr=28 luod=0'0 crt=29'9 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.112479012s 2019-12-07T19:34:14.657+0000 7fe2cb26d700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease_ack(1.1s0 pg_lease_ack(ruub 54.115157360s) e37/37) v1 -- 0x558e8403e780 con 0x558e818d7000 2019-12-07T19:34:14.657+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 267 ==== pg_lease(1.es4 pg_lease(ru 46.114825888s ub 54.114968172s int 16s) e37/37) v1 ==== 58+0+0 (unknown 0 0 0) 0x558e83fee000 con 0x558e818d7c00 2019-12-07T19:34:14.657+0000 7fe2cb26d700 10 log is not dirty 2019-12-07T19:34:14.657+0000 7fe2e7d3c700 15 osd.0 37 enqueue_peering_evt 1.es4 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114825888s ub 54.114968172s int 16s) 2019-12-07T19:34:14.657+0000 7fe2cb26d700 20 osd.0 op_wq(1) _process empty q, waiting 2019-12-07T19:34:14.657+0000 7fe2e7d3c700 20 osd.0 op_wq(6) _enqueue OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114825888s ub 54.114968172s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.657+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.657+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114825888s ub 54.114968172s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.657+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process 1.es4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.657+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process OpSchedulerItem(1.es4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114825888s ub 54.114968172s int 16s)) prio 255 cost 10 e37) pg 0x558e81a98000 2019-12-07T19:34:14.657+0000 7fe2c4a60700 10 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.114825888s ub 54.114968172s int 16s) 2019-12-07T19:34:14.657+0000 7fe2c4a60700 10 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.114825888s ub 54.114968172s int 16s) 2019-12-07T19:34:14.657+0000 7fe2c4a60700 20 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] peer_clock_delta_ub 0.002416414s -> ru 46.112409474s 2019-12-07T19:34:14.657+0000 7fe2c4a60700 20 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] proc_lease readable_until now 46.112409474s 2019-12-07T19:34:14.657+0000 7fe2c4a60700 20 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] peer_clock_delta_lb 0.002061166s -> ruub 54.112907006s 2019-12-07T19:34:14.657+0000 7fe2c4a60700 20 osd.0 pg_epoch: 37 pg[1.es4( v 21'4 (0'0,21'4] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,3,4,5,0,2]p1(0) r=4 lpr=28 luod=0'0 crt=21'4 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.112907006s 2019-12-07T19:34:14.657+0000 7fe2c4a60700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.es0 pg_lease_ack(ruub 54.114968172s) e37/37) v1 -- 0x558e819b0d20 con 0x558e818d7c00 2019-12-07T19:34:14.657+0000 7fe2c4a60700 10 log is not dirty 2019-12-07T19:34:14.657+0000 7fe2c4a60700 20 osd.0 op_wq(6) _process empty q, waiting 2019-12-07T19:34:14.657+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 268 ==== pg_lease(1.as3 pg_lease(ru 46.115034256s ub 54.115283607s int 16s) e37/37) v1 ==== 58+0+0 (unknown 0 0 0) 0x558e83fee000 con 0x558e818d7c00 2019-12-07T19:34:14.657+0000 7fe2e7d3c700 15 osd.0 37 enqueue_peering_evt 1.as3 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.115034256s ub 54.115283607s int 16s) 2019-12-07T19:34:14.657+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.115034256s ub 54.115283607s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.657+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.657+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.115034256s ub 54.115283607s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.657+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.as3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.657+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.as3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.115034256s ub 54.115283607s int 16s)) prio 255 cost 10 e37) pg 0x558e8193b000 2019-12-07T19:34:14.657+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.115034256s ub 54.115283607s int 16s) 2019-12-07T19:34:14.657+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.115034256s ub 54.115283607s int 16s) 2019-12-07T19:34:14.657+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] peer_clock_delta_ub 0.002416414s -> ru 46.112617842s 2019-12-07T19:34:14.657+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] proc_lease readable_until now 46.112617842s 2019-12-07T19:34:14.657+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] peer_clock_delta_lb 0.002061166s -> ruub 54.113222441s 2019-12-07T19:34:14.657+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.as3( v 21'2 (0'0,21'2] local-lis/les=28/29 n=1 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,2,0,5,3]p1(0) r=3 lpr=28 luod=0'0 crt=21'2 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.113222441s 2019-12-07T19:34:14.657+0000 7fe2caa6c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.as0 pg_lease_ack(ruub 54.115283607s) e37/37) v1 -- 0x558e819c4f00 con 0x558e818d7c00 2019-12-07T19:34:14.657+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:14.657+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:14.658+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 269 ==== pg_lease(1.7s3 pg_lease(ru 46.116162602s ub 54.116297009s int 16s) e37/37) v1 ==== 58+0+0 (unknown 0 0 0) 0x558e83fee000 con 0x558e818d7c00 2019-12-07T19:34:14.658+0000 7fe2e7d3c700 15 osd.0 37 enqueue_peering_evt 1.7s3 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.116162602s ub 54.116297009s int 16s) 2019-12-07T19:34:14.658+0000 7fe2e7d3c700 20 osd.0 op_wq(7) _enqueue OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.116162602s ub 54.116297009s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.114604348s ub 54.115092785s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.fs1 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.fs1 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.114604348s ub 54.115092785s int 16s)) prio 255 cost 10 e37) pg 0x558e81a9b000 2019-12-07T19:34:14.659+0000 7fe2c425f700 10 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.3 pg_lease(ru 46.114604348s ub 54.115092785s int 16s) 2019-12-07T19:34:14.659+0000 7fe2c425f700 10 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.114604348s ub 54.115092785s int 16s) 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] peer_clock_delta_ub 0.002985006s -> ru 46.111619342s 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] proc_lease readable_until now 46.111619342s 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] peer_clock_delta_lb 0.002678348s -> ruub 54.112414437s 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 pg_epoch: 37 pg[1.fs1( v 29'12 (0'0,29'12] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [3,0,5,4,1,2]p3(0) r=1 lpr=28 luod=0'0 crt=29'12 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.112414437s 2019-12-07T19:34:14.659+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease_ack(1.fs0 pg_lease_ack(ruub 54.115092785s) e37/37) v1 -- 0x558e847385a0 con 0x558e818d7000 2019-12-07T19:34:14.659+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.116162602s ub 54.116297009s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process 1.7s3 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process OpSchedulerItem(1.7s3 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.116162602s ub 54.116297009s int 16s)) prio 255 cost 10 e37) pg 0x558e8122e000 2019-12-07T19:34:14.659+0000 7fe2c425f700 10 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.1 pg_lease(ru 46.116162602s ub 54.116297009s int 16s) 2019-12-07T19:34:14.659+0000 7fe2c425f700 10 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.116162602s ub 54.116297009s int 16s) 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] peer_clock_delta_ub 0.002416414s -> ru 46.113746188s 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] proc_lease readable_until now 46.113746188s 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] peer_clock_delta_lb 0.002061166s -> ruub 54.114235843s 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 pg_epoch: 37 pg[1.7s3( v 27'10 (0'0,27'10] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [1,4,3,0,7,2]p1(0) r=3 lpr=28 luod=0'0 crt=27'10 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.114235843s 2019-12-07T19:34:14.659+0000 7fe2c425f700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease_ack(1.7s0 pg_lease_ack(ruub 54.116297009s) e37/37) v1 -- 0x558e84738780 con 0x558e818d7c00 2019-12-07T19:34:14.659+0000 7fe2c425f700 10 log is not dirty 2019-12-07T19:34:14.659+0000 7fe2c425f700 20 osd.0 op_wq(7) _process empty q, waiting 2019-12-07T19:34:14.660+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.2 v2:172.21.15.131:6804/37077 54 ==== pg_lease(1.5s4 pg_lease(ru 46.131453992s ub 54.132455501s int 16s) e37/37) v1 ==== 58+0+0 (unknown 0 0 0) 0x558e818efa00 con 0x558e818d6000 2019-12-07T19:34:14.660+0000 7fe2e753b700 15 osd.0 37 enqueue_peering_evt 1.5s4 epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.2 pg_lease(ru 46.131453992s ub 54.132455501s int 16s) 2019-12-07T19:34:14.660+0000 7fe2e753b700 20 osd.0 op_wq(5) _enqueue OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.2 pg_lease(ru 46.131453992s ub 54.132455501s int 16s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.662+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.662+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.2 pg_lease(ru 46.131453992s ub 54.132455501s int 16s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.662+0000 7fe2c5261700 20 osd.0 op_wq(5) _process 1.5s4 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.662+0000 7fe2c5261700 20 osd.0 op_wq(5) _process OpSchedulerItem(1.5s4 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.2 pg_lease(ru 46.131453992s ub 54.132455501s int 16s)) prio 255 cost 10 e37) pg 0x558e83d07000 2019-12-07T19:34:14.662+0000 7fe2c5261700 10 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLease epoch 37 from osd.2 pg_lease(ru 46.131453992s ub 54.132455501s int 16s) 2019-12-07T19:34:14.662+0000 7fe2c5261700 10 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] proc_lease pg_lease(ru 46.131453992s ub 54.132455501s int 16s) 2019-12-07T19:34:14.662+0000 7fe2c5261700 20 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] peer_clock_delta_ub 0.017206124s -> ru 46.114247868s 2019-12-07T19:34:14.662+0000 7fe2c5261700 20 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] proc_lease readable_until now 46.114247868s 2019-12-07T19:34:14.662+0000 7fe2c5261700 20 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] peer_clock_delta_lb 0.016829204s -> ruub 54.115626297s 2019-12-07T19:34:14.662+0000 7fe2c5261700 20 osd.0 pg_epoch: 37 pg[1.5s4( v 27'8 (0'0,27'8] local-lis/les=28/29 n=3 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [2,1,3,4,0,5]p2(0) r=4 lpr=28 luod=0'0 crt=27'8 active mbc={} ps=[a~1]] proc_lease readable_until_ub now 54.115626297s 2019-12-07T19:34:14.662+0000 7fe2c5261700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6804/37077,v1:172.21.15.131:6805/37077] -- pg_lease_ack(1.5s0 pg_lease_ack(ruub 54.132455501s) e37/37) v1 -- 0x558e847385a0 con 0x558e818d6000 2019-12-07T19:34:14.662+0000 7fe2c5261700 10 log is not dirty 2019-12-07T19:34:14.662+0000 7fe2c5261700 20 osd.0 op_wq(5) _process empty q, waiting 2019-12-07T19:34:14.663+0000 7fe2cca70700 15 osd.0 37 enqueue_peering_evt 1.2s0 epoch_sent: 28 epoch_requested: 28 RenewLease 2019-12-07T19:34:14.663+0000 7fe2cca70700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) 2019-12-07T19:34:14.663+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.663+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) queued 2019-12-07T19:34:14.663+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.663+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 28 epoch_requested: 28 RenewLease) prio 255 cost 10 e28) pg 0x558e81075000 2019-12-07T19:34:14.663+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_peering_event: epoch_sent: 28 epoch_requested: 28 RenewLease 2019-12-07T19:34:14.663+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.4 ruub 46.119144108s 2019-12-07T19:34:14.663+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.1 ruub 46.119144108s 2019-12-07T19:34:14.663+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.5 ruub 46.119144108s 2019-12-07T19:34:14.663+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.7 ruub 46.119144108s 2019-12-07T19:34:14.663+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.3 ruub 46.119144108s 2019-12-07T19:34:14.663+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until readable_until[_ub] 46.119144108s (sent 54.119305575s) 2019-12-07T19:34:14.663+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6823/37078,v1:172.21.15.131:6825/37078] -- pg_lease(1.2s2 pg_lease(ru 46.119144108s ub 54.119305575s int 16s) e37/37) v1 -- 0x558e811f6c00 con 0x558e818d7c00 2019-12-07T19:34:14.663+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.131:6812/37076,v1:172.21.15.131:6813/37076] -- pg_lease(1.2s5 pg_lease(ru 46.119144108s ub 54.119305575s int 16s) e37/37) v1 -- 0x558e811f6e00 con 0x558e818d7000 2019-12-07T19:34:14.663+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6810/36938,v1:172.21.15.109:6811/36938] -- pg_lease(1.2s1 pg_lease(ru 46.119144108s ub 54.119305575s int 16s) e37/37) v1 -- 0x558e811f6800 con 0x558e818fe400 2019-12-07T19:34:14.663+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6802/36936,v1:172.21.15.109:6803/36936] -- pg_lease(1.2s3 pg_lease(ru 46.119144108s ub 54.119305575s int 16s) e37/37) v1 -- 0x558e811f7800 con 0x558e818d6800 2019-12-07T19:34:14.663+0000 7fe2c6a64700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] --> [v2:172.21.15.109:6826/36939,v1:172.21.15.109:6827/36939] -- pg_lease(1.2s4 pg_lease(ru 46.119144108s ub 54.119305575s int 16s) e37/37) v1 -- 0x558e81955800 con 0x558e818fe000 2019-12-07T19:34:14.664+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:14.664+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.3 v2:172.21.15.131:6812/37076 138 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 54.119305575s) e37/37) v1 ==== 42+0+0 (unknown 0 0 0) 0x558e84738780 con 0x558e818d7000 2019-12-07T19:34:14.664+0000 7fe2e7d3c700 15 osd.0 37 enqueue_peering_evt 1.2s0 epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.3 pg_lease_ack(ruub 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.3 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.3 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.3 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) pg 0x558e81075000 2019-12-07T19:34:14.664+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.1 v2:172.21.15.131:6823/37078 270 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 54.119305575s) e37/37) v1 ==== 42+0+0 (unknown 0 0 0) 0x558e84738780 con 0x558e818d7c00 2019-12-07T19:34:14.664+0000 7fe2e7d3c700 15 osd.0 37 enqueue_peering_evt 1.2s0 epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.1 pg_lease_ack(ruub 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2e7d3c700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.1 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.3 pg_lease_ack(ruub 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.7 v2:172.21.15.109:6826/36939 47 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 54.119305575s) e37/37) v1 ==== 42+0+0 (unknown 0 0 0) 0x558e847385a0 con 0x558e818fe000 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.4 ruub 46.119144108s 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.1 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.664+0000 7fe2e753b700 15 osd.0 37 enqueue_peering_evt 1.2s0 epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.7 pg_lease_ack(ruub 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.1 ruub 46.119144108s 2019-12-07T19:34:14.664+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.7 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.5 ruub 46.119144108s 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.7 ruub 46.119144108s 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.3 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until readable_until[_ub] 46.119144108s (sent 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2e753b700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.5 v2:172.21.15.109:6802/36936 59 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 54.119305575s) e37/37) v1 ==== 42+0+0 (unknown 0 0 0) 0x558e847385a0 con 0x558e818d6800 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:14.664+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:14.664+0000 7fe2e753b700 15 osd.0 37 enqueue_peering_evt 1.2s0 epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.5 pg_lease_ack(ruub 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.664+0000 7fe2e753b700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.5 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.664+0000 7fe2e6d3a700 1 -- [v2:172.21.15.131:6822/37079,v1:172.21.15.131:6824/37079] <== osd.4 v2:172.21.15.109:6810/36938 142 ==== pg_lease_ack(1.2s0 pg_lease_ack(ruub 54.119305575s) e37/37) v1 ==== 42+0+0 (unknown 0 0 0) 0x558e8122de00 con 0x558e818fe400 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.1 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) pg 0x558e81075000 2019-12-07T19:34:14.664+0000 7fe2e6d3a700 15 osd.0 37 enqueue_peering_evt 1.2s0 epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.4 pg_lease_ack(ruub 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2e6d3a700 20 osd.0 op_wq(2) _enqueue OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.4 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.664+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.1 pg_lease_ack(ruub 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.7 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.4 ruub 46.119144108s 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.1 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.5 ruub 46.119144108s 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.7 ruub 46.119144108s 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.3 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until readable_until[_ub] 46.119144108s (sent 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:14.664+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.7 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) pg 0x558e81075000 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process <> waiting <> waiting_peering {} 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.5 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.7 pg_lease_ack(ruub 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.4 ruub 46.119144108s 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.1 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.5 ruub 46.119144108s 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.7 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.3 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until readable_until[_ub] 46.119144108s (sent 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:14.664+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.4 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) queued 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.5 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) pg 0x558e81075000 2019-12-07T19:34:14.664+0000 7fe2c6a64700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.5 pg_lease_ack(ruub 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.4 ruub 46.119144108s 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.1 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.5 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.7 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.3 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until readable_until[_ub] 46.119144108s (sent 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:14.664+0000 7fe2c6a64700 10 log is not dirty 2019-12-07T19:34:14.664+0000 7fe2c6a64700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process 1.2s0 to_process waiting <> waiting_peering {} 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process OpSchedulerItem(1.2s0 PGPeeringEvent(epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.4 pg_lease_ack(ruub 54.119305575s)) prio 255 cost 10 e37) pg 0x558e81075000 2019-12-07T19:34:14.664+0000 7fe2caa6c700 10 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] do_peering_event: epoch_sent: 37 epoch_requested: 37 MLeaseAck epoch 37 from osd.4 pg_lease_ack(ruub 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.4 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.1 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.5 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.7 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until peer osd.3 ruub 54.119305575s 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recalc_readable_until readable_until[_ub] 54.119305575s (sent 54.119305575s) 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 pg_epoch: 37 pg[1.2s0( v 29'15 (0'0,29'15] local-lis/les=28/29 n=2 ec=18/18 lis/c=28/28 les/c/f=29/29/0 sis=28) [0,4,1,5,7,3]p0(0) r=0 lpr=28 crt=29'15 lcod 29'13 mlcod 29'13 active+clean ps=[a~1]] recheck_readable wasn't wait or laggy 2019-12-07T19:34:14.664+0000 7fe2caa6c700 10 log is not dirty 2019-12-07T19:34:14.664+0000 7fe2caa6c700 20 osd.0 op_wq(2) _process empty q, waiting 2019-12-07T19:34:14.684+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 710541 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.734+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 710541 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.784+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 710541 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.835+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 710541 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.844+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] <== osd.5 172.21.15.109:0/36936 13 ==== osd_ping(ping e37 up_from 13 ping_stamp 2019-12-07T19:34:14.844723+0000/38.311047011s send_stamp 38.311047011s delta_ub 0.011820728s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83fee000 con 0x558e8182c800 2019-12-07T19:34:14.844+0000 7fe2e7d3c700 20 osd.0 37 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011384147s,0.011820728s]) 2019-12-07T19:34:14.844+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6826/37079,v1:172.21.15.131:6828/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e37 up_from 13 ping_stamp 2019-12-07T19:34:14.844723+0000/38.311047011s send_stamp 38.299662864s delta_ub 18446744073.698167469s) v5 -- 0x558e811f6c00 con 0x558e8182c800 2019-12-07T19:34:14.844+0000 7fe2e7d3c700 10 osd.0 37 maybe_share_map con 0x558e818d6800 v2:172.21.15.109:6802/36936 map epoch 33 -> 37 (as per caller) 2019-12-07T19:34:14.844+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] <== osd.5 172.21.15.109:0/36936 13 ==== osd_ping(ping e37 up_from 13 ping_stamp 2019-12-07T19:34:14.844723+0000/38.311047011s send_stamp 38.311047011s delta_ub 0.011820728s) v5 ==== 2033+0+0 (crc 0 0 0) 0x558e83fee000 con 0x558e8181b800 2019-12-07T19:34:14.844+0000 7fe2e7d3c700 20 osd.0 37 handle_osd_ping new stamps hbstamp(osd.5 up_from 13 peer_clock_delta [0.011335723s,0.011820728s]) 2019-12-07T19:34:14.844+0000 7fe2e7d3c700 1 -- [v2:172.21.15.131:6830/37079,v1:172.21.15.131:6832/37079] --> 172.21.15.109:0/36936 -- osd_ping(ping_reply e37 up_from 13 ping_stamp 2019-12-07T19:34:14.844723+0000/38.311047011s send_stamp 38.299711288s delta_ub 18446744073.698215893s) v5 -- 0x558e84052c00 con 0x558e8181b800 2019-12-07T19:34:14.850+0000 7fe2e4d36700 -1 received signal: Hangup from /usr/bin/python /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 37069) UID: 0 2019-12-07T19:34:14.873+0000 7fe2e7d3c700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e843a2400 0x558e81a38580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rx=0 tx=0).accept 2019-12-07T19:34:14.873+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> conn(0x558e843a2400 0x558e81a38580 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rx=0 tx=0)._handle_peer_banner_payload supported=0 required=0 2019-12-07T19:34:14.873+0000 7fe2e6d3a700 10 osd.0 37 ms_handle_authentication new session 0x558e83ff6340 con 0x558e843a2400 entity client.admin addr 2019-12-07T19:34:14.873+0000 7fe2e6d3a700 10 osd.0 37 ms_handle_authentication session 0x558e83ff6340 client.admin has caps osdcap[grant(*)] 'allow *' 2019-12-07T19:34:14.873+0000 7fe2e6d3a700 1 --2- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] >> 172.21.15.131:0/2098596510 conn(0x558e843a2400 0x558e81a38580 crc :-1 s=READY pgs=5 cs=0 l=1 rx=0 tx=0).ready entity=client.4454 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2019-12-07T19:34:14.873+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4454 172.21.15.131:0/2098596510 1 ==== command(tid 1: {"prefix": "get_command_descriptions"}) v1 ==== 62+0+0 (crc 0 0 0) 0x558e8122de00 con 0x558e843a2400 2019-12-07T19:34:14.873+0000 7fe2d959f700 20 osd.0 37 OSD::ms_dispatch: command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:14.873+0000 7fe2d959f700 10 osd.0 37 do_waiters -- start 2019-12-07T19:34:14.873+0000 7fe2d959f700 10 osd.0 37 do_waiters -- finish 2019-12-07T19:34:14.873+0000 7fe2d959f700 20 osd.0 37 _dispatch 0x558e8122de00 command(tid 1: {"prefix": "get_command_descriptions"}) v1 2019-12-07T19:34:14.874+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/2098596510 -- command_reply(tid 1: 0 ) v1 -- 0x558e83c83880 con 0x558e843a2400 2019-12-07T19:34:14.885+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 710541 data_alloc: 671088640 data_used: 0 2019-12-07T19:34:14.894+0000 7fe2d959f700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] <== client.4454 172.21.15.131:0/2098596510 2 ==== command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 ==== 99+0+0 (crc 0 0 0) 0x558e84738f00 con 0x558e843a2400 2019-12-07T19:34:14.894+0000 7fe2d959f700 20 osd.0 37 OSD::ms_dispatch: command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:14.894+0000 7fe2d959f700 10 osd.0 37 do_waiters -- start 2019-12-07T19:34:14.894+0000 7fe2d959f700 10 osd.0 37 do_waiters -- finish 2019-12-07T19:34:14.894+0000 7fe2d959f700 20 osd.0 37 _dispatch 0x558e84738f00 command(tid 2: {"injected_args": ["--osd_enable_op_tracker=true"], "prefix": "injectargs"}) v1 2019-12-07T19:34:14.896+0000 7fe2e5d38700 1 -- [v2:172.21.15.131:6818/37079,v1:172.21.15.131:6820/37079] --> 172.21.15.131:0/2098596510 -- command_reply(tid 2: 0 osd_enable_op_tracker = 'true' ) v1 -- 0x558e83c836c0 con 0x558e843a2400 2019-12-07T19:34:14.935+0000 7fe2dc28f700 20 bluestore.MempoolThread(0x558e80f12a08) _resize_shards cache_size: 2845415832 kv_alloc: 1073741824 kv_used: 1760 meta_alloc: 1073741824 meta_used: 710541 data_alloc: 671088640 data_used: 0